From 2a82016eb049904b8f3de2de7b94382af157b82a Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Sat, 24 Oct 2026 16:41:52 -0500 Subject: [PATCH 1/4] feat(passthrough): let upstreams report the model name via x-litellm-upstream-model A pass-through upstream that prices its own requests already reports cost and tokens in x-litellm-response-cost / x-litellm-total-tokens. Product endpoints that take no `model` in the body still landed under `unknown`, so their spend could not be broken down. When the upstream sends x-litellm-upstream-model, record its value as the request's model on success and failure rows, overriding any body `model`. The name is a label only: it never marks the request as upstream-priced and is not used to look up a price. A blank or missing header changes nothing. Refs LIT-9910 --- .../pass_through_endpoints.py | 17 +++++ .../upstream_usage_headers.py | 36 ++++++++- .../test_pass_through_endpoints.py | 74 +++++++++++++++++-- .../test_upstream_usage_headers.py | 65 +++++++++++++++- 4 files changed, 184 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e1e8864678e..85534ec6c60 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -148,6 +148,7 @@ from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging from .upstream_usage_headers import ( UpstreamReportedUsage, + apply_upstream_reported_model, apply_upstream_reported_usage, ) @@ -852,6 +853,7 @@ def _build_passthrough_failure_request_payload( logging_obj: LiteLLMLoggingObj | None, custom_llm_provider: str | None, upstream_usage: UpstreamReportedUsage | None = None, + upstream_model: str | None = None, ) -> dict: """Build the ``request_data`` dict passed to ``post_call_failure_hook``. @@ -862,6 +864,8 @@ def _build_passthrough_failure_request_payload( ``upstream_usage`` carries the cost and tokens an upstream reported on an error response. Spend tracking only attributes a recovered cost when it comes paired with a usage object, so both keys are written together. + ``upstream_model`` is the model name the upstream reported, which replaces + the body's ``model`` on the failure row just as it does on a success row. """ request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body if kwargs: @@ -875,6 +879,8 @@ def _build_passthrough_failure_request_payload( if upstream_usage is not None: request_payload["response_cost"] = upstream_usage.response_cost or 0.0 request_payload["combined_usage_object"] = Usage(total_tokens=upstream_usage.total_tokens or 0) + if upstream_model is not None: + request_payload["model"] = upstream_model return request_payload @@ -1538,6 +1544,10 @@ async def pass_through_request( logging_obj=logging_obj, headers=response.headers, ) + upstream_model = apply_upstream_reported_model( # rebind-ok: bound once per branch, like upstream_usage + logging_obj=logging_obj, + headers=response.headers, + ) relay_response: Final = await _log_passthrough_upstream_failure( response=response, @@ -1548,6 +1558,7 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + upstream_model=upstream_model, ), logging_obj=logging_obj, ) @@ -1626,6 +1637,10 @@ async def pass_through_request( logging_obj=logging_obj, headers=response.headers, ) + upstream_model = apply_upstream_reported_model( # rebind-ok: bound once per branch, like upstream_usage + logging_obj=logging_obj, + headers=response.headers, + ) if _is_streaming_response(response) is True: logging_obj.stream = True @@ -1640,6 +1655,7 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + upstream_model=upstream_model, ), logging_obj=logging_obj, ) @@ -1740,6 +1756,7 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + upstream_model=upstream_model, ) failure_request_payload["response_body"] = response_body await _log_passthrough_upstream_failure( diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index 59b8d40577a..870d5438eed 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -3,7 +3,9 @@ Some pass-through targets invoke several models internally, so LiteLLM cannot price the request from the response body. Those targets report the totals for the whole HTTP request in ``x-litellm-*`` response headers instead; LiteLLM -records the reported values without recomputing them. +records the reported values without recomputing them. A target may also name +the request in ``x-litellm-upstream-model`` so its spend can be told apart from +other requests that carry no ``model`` in the body. """ import math @@ -18,6 +20,9 @@ from litellm.types.utils import Usage UPSTREAM_RESPONSE_COST_HEADER: Final = "x-litellm-response-cost" UPSTREAM_TOTAL_TOKENS_HEADER: Final = "x-litellm-total-tokens" +# Deliberately not x-litellm-model-name/-id/-group: the gateway already sends +# those to clients, and pass-through relays upstream headers back to the client. +UPSTREAM_MODEL_HEADER: Final = "x-litellm-upstream-model" # model_call_details key holding what the upstream reported, so later stages of # the success path can tell an upstream-reported cost apart from one LiteLLM @@ -127,6 +132,35 @@ def apply_upstream_reported_usage( return reported +def parse_upstream_reported_model(headers: httpx.Headers) -> str | None: + """Read the model name an upstream reported, or ``None`` when it sent none. + + An empty or whitespace-only value counts as absent. + """ + raw_model: Final[str | None] = headers.get(UPSTREAM_MODEL_HEADER) + model: Final = raw_model.strip() if raw_model is not None else "" + return model or None + + +def apply_upstream_reported_model( + logging_obj: LiteLLMLoggingObj, + headers: httpx.Headers, +) -> str | None: + """Record the upstream's model name as the request's model. + + The name wins over a ``model`` in the request body. It is a label only: + it never marks the request as upstream-priced (see + ``has_upstream_reported_usage``) and nothing prices the request from it, + so a name outside the price map leaves the recorded cost alone. + """ + reported_model: Final = parse_upstream_reported_model(headers) + if reported_model is None: + return None + logging_obj.model = reported_model # rebind-ok: the request's logging object is where the recorded model lives + logging_obj.model_call_details["model"] = reported_model # rebind-ok: read by the standard logging payload + return reported_model + + def has_upstream_reported_usage(logging_obj: LiteLLMLoggingObj) -> bool: """Whether the upstream spoke this contract on the request's response. diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index db0a74d360a..9e30f0b5aca 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -6726,6 +6726,7 @@ async def _run_upstream_reporting_passthrough( status_code: int = 200, cost_per_request: float | None = None, request_headers: Mapping[str, str] | None = None, + request_body: Mapping[str, object] | None = None, ) -> tuple[list[object], MagicMock]: """Drive a generic pass-through against an upstream that reports its own cost/usage. Returns (recorded standard logging payloads, proxy logging mock).""" @@ -6741,13 +6742,14 @@ async def _run_upstream_reporting_passthrough( ) try: with ExitStack() as stack: - mock_proxy_logging, enqueued = _enter_upstream_usage_mocks(stack, {}) + mock_proxy_logging, enqueued = _enter_upstream_usage_mocks(stack, dict(request_body or {})) with _recording_success_callback() as recorder: await pass_through_request( request=_relay_client_request(method="POST", headers=request_headers), target="http://internal-api.test/v1/summarize", custom_headers={}, user_api_key_dict=UserAPIKeyAuth(api_key="sk-upstream-usage", team_id="team-fil"), + custom_body=dict(request_body) if request_body else None, cost_per_request=cost_per_request, ) for coroutine in enqueued: @@ -6846,12 +6848,72 @@ async def test_passthrough_error_response_without_usage_headers_records_no_spend assert "combined_usage_object" not in request_data +_UPSTREAM_SUMMARIZER_HEADERS: Final = MappingProxyType( + { + "x-litellm-response-cost": "0.0123", + "x-litellm-total-tokens": "4210", + "x-litellm-upstream-model": "support-summarizer", + } +) + + @pytest.mark.asyncio -async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstream(): +@pytest.mark.parametrize("request_body", [None, {"model": "gpt-4o"}], ids=["no_body_model", "body_model"]) +async def test_passthrough_records_model_reported_by_upstream(request_body: Mapping[str, object] | None): + """A product endpoint takes no `model`, so without the header its spend lands + under `unknown`. The upstream's name wins over a body `model` too, and the + reported cost is kept even though the name is not in the price map.""" + payloads, _ = await _run_upstream_reporting_passthrough(_UPSTREAM_SUMMARIZER_HEADERS, request_body=request_body) + + assert len(payloads) == 1 + logged = cast(types_utils.StandardLoggingPayload, payloads[0]) + assert (logged["model"], logged["response_cost"], logged["total_tokens"]) == ( + "support-summarizer", + 0.0123, + 4210, + ), logged + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_body", "expected_model"), + [(None, "unknown"), ({"model": "gpt-4o"}, "gpt-4o")], + ids=["no_body_model", "body_model"], +) +async def test_passthrough_without_model_header_keeps_request_model( + request_body: Mapping[str, object] | None, expected_model: str +): + payloads, _ = await _run_upstream_reporting_passthrough( + {"x-litellm-response-cost": "0.0123", "x-litellm-total-tokens": "4210"}, request_body=request_body + ) + + assert len(payloads) == 1 + assert cast(types_utils.StandardLoggingPayload, payloads[0])["model"] == expected_model, payloads[0] + + +@pytest.mark.asyncio +async def test_passthrough_records_upstream_reported_model_on_error_response(): + import litellm + + _, mock_proxy_logging = await _run_upstream_reporting_passthrough( + _UPSTREAM_SUMMARIZER_HEADERS, status_code=500, request_body={"model": "gpt-4o"} + ) + + mock_proxy_logging.post_call_failure_hook.assert_awaited_once() + request_data = mock_proxy_logging.post_call_failure_hook.await_args.kwargs["request_data"] + assert (request_data["model"], request_data["response_cost"], request_data["combined_usage_object"]) == ( + "support-summarizer", + 0.0123, + litellm.Usage(total_tokens=4210), + ), request_data + + +@pytest.mark.asyncio +async def test_streaming_passthrough_records_cost_tokens_and_model_reported_by_upstream(): """ - The totals are reported in the response headers, which are known before the - first byte of the stream, so a streamed pass-through must record the same - spend a buffered one does. + The totals and model name are reported in the response headers, which are + known before the first byte of the stream, so a streamed pass-through must + record the same spend and model a buffered one does. """ from fastapi.responses import StreamingResponse @@ -6864,6 +6926,7 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea "content-type": "text/event-stream", "x-litellm-response-cost": "0.00312", "x-litellm-total-tokens": "4021", + "x-litellm-upstream-model": "support-summarizer", }, stream=_RecordingUpstreamByteStream((b'data: {"delta": "hi"}\n\n',)), ), @@ -6890,6 +6953,7 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea assert len(recorder.payloads) == 1 assert recorder.payloads[0]["response_cost"] == 0.00312 assert recorder.payloads[0]["total_tokens"] == 4021 + assert recorder.payloads[0]["model"] == "support-summarizer" @pytest.mark.asyncio diff --git a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py index 701c583a4ab..0fef091acae 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py +++ b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py @@ -1,12 +1,13 @@ - import httpx import pytest - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.upstream_usage_headers import ( UpstreamReportedUsage, + apply_upstream_reported_model, apply_upstream_reported_usage, + has_upstream_reported_usage, + parse_upstream_reported_model, parse_upstream_reported_usage, ) from litellm.types.utils import Usage @@ -126,3 +127,63 @@ def test_apply_only_overwrites_what_upstream_reported(): assert logging_obj.model_call_details["response_cost"] == 0.5 assert logging_obj.model_call_details["combined_usage_object"] == Usage(total_tokens=42) + + +@pytest.mark.parametrize( + ("raw_model", "expected"), + [ + ("support-summarizer", "support-summarizer"), + (" support-summarizer \t", "support-summarizer"), + ("", None), + (" ", None), + ], +) +def test_parse_reads_reported_model_and_treats_blank_as_absent(raw_model: str, expected: str | None): + assert parse_upstream_reported_model(_headers(**{"x-litellm-upstream-model": raw_model})) == expected + + +def test_parse_returns_no_model_when_upstream_sends_none(): + assert parse_upstream_reported_model(_headers(**{"x-litellm-response-cost": "0.5"})) is None + + +def _logging_obj_for_model(model: str) -> LiteLLMLoggingObj: + logging_obj = _logging_obj() + logging_obj.update_environment_variables( + model=model, user="unknown", optional_params={}, litellm_params={}, call_type="pass_through_endpoint" + ) + return logging_obj + + +def test_apply_model_replaces_the_request_model(): + logging_obj = _logging_obj_for_model("gpt-4o") + + reported = apply_upstream_reported_model( + logging_obj=logging_obj, + headers=_headers(**{"x-litellm-upstream-model": "support-summarizer"}), + ) + + assert reported == "support-summarizer" + assert (logging_obj.model, logging_obj.model_call_details["model"]) == ("support-summarizer", "support-summarizer") + + +def test_apply_model_leaves_request_model_alone_when_header_is_blank(): + logging_obj = _logging_obj_for_model("gpt-4o") + + assert ( + apply_upstream_reported_model(logging_obj=logging_obj, headers=_headers(**{"x-litellm-upstream-model": " "})) + is None + ) + assert (logging_obj.model, logging_obj.model_call_details["model"]) == ("gpt-4o", "gpt-4o") + + +def test_model_header_alone_does_not_make_the_upstream_own_the_cost(): + """The model name is a label: without a cost or token header the request + must still fall back to cost_per_request, not be recorded at 0.""" + logging_obj = _logging_obj() + headers = _headers(**{"x-litellm-upstream-model": "support-summarizer"}) + + apply_upstream_reported_usage(logging_obj=logging_obj, headers=headers) + apply_upstream_reported_model(logging_obj=logging_obj, headers=headers) + + assert has_upstream_reported_usage(logging_obj) is False + assert "response_cost" not in logging_obj.model_call_details From 39f90179e262e3582d7a1191e64bb667ff661ce4 Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Sat, 24 Oct 2026 20:18:42 -0500 Subject: [PATCH 2/4] fix(passthrough): keep the upstream-reported model name bounded The upstream model name becomes a spend row, a daily rollup key and a metric label. Use only the first element of a repeated header (RFC 9110 field combining, which is how httpx joins it) and cut the name to MAX_SPEND_LOG_MODEL_NAME_LENGTH, the limit the spend log already applies to model names. Refs LIT-9910 --- .../upstream_usage_headers.py | 11 +++++++--- .../test_upstream_usage_headers.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index 870d5438eed..22e6f1797c7 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -15,6 +15,7 @@ from typing import Final import httpx from litellm._logging import verbose_proxy_logger +from litellm.constants import MAX_SPEND_LOG_MODEL_NAME_LENGTH from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.utils import Usage @@ -135,11 +136,15 @@ def apply_upstream_reported_usage( def parse_upstream_reported_model(headers: httpx.Headers) -> str | None: """Read the model name an upstream reported, or ``None`` when it sent none. - An empty or whitespace-only value counts as absent. + An empty or whitespace-only value counts as absent. A repeated header reads + as one comma-separated list (RFC 9110 field combining, which is how httpx + joins it), so only its first element is used. The name is cut to the spend + log's model-name limit, since it becomes a spend row, rollup key and metric + label. """ raw_model: Final[str | None] = headers.get(UPSTREAM_MODEL_HEADER) - model: Final = raw_model.strip() if raw_model is not None else "" - return model or None + model: Final = raw_model.split(",", 1)[0].strip() if raw_model is not None else "" + return model[:MAX_SPEND_LOG_MODEL_NAME_LENGTH].rstrip() or None def apply_upstream_reported_model( diff --git a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py index 0fef091acae..f0b8ffd3664 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py +++ b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py @@ -1,6 +1,7 @@ import httpx import pytest +from litellm.constants import MAX_SPEND_LOG_MODEL_NAME_LENGTH from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.upstream_usage_headers import ( UpstreamReportedUsage, @@ -146,6 +147,25 @@ def test_parse_returns_no_model_when_upstream_sends_none(): assert parse_upstream_reported_model(_headers(**{"x-litellm-response-cost": "0.5"})) is None +def test_parse_uses_the_first_value_of_a_repeated_model_header(): + headers = httpx.Headers([("x-litellm-upstream-model", "support-summarizer"), ("x-litellm-upstream-model", "other")]) + + assert parse_upstream_reported_model(headers) == "support-summarizer" + + +@pytest.mark.parametrize( + ("raw_model", "expected"), + [ + ("m" * 5000, "m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH), + ("m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH, "m" * MAX_SPEND_LOG_MODEL_NAME_LENGTH), + ("m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH - 1) + " tail", "m" * (MAX_SPEND_LOG_MODEL_NAME_LENGTH - 1)), + ], + ids=["over_limit", "at_limit", "cut_lands_on_a_space"], +) +def test_parse_cuts_a_long_model_name_to_the_spend_log_limit(raw_model: str, expected: str): + assert parse_upstream_reported_model(_headers(**{"x-litellm-upstream-model": raw_model})) == expected + + def _logging_obj_for_model(model: str) -> LiteLLMLoggingObj: logging_obj = _logging_obj() logging_obj.update_environment_variables( From 11c51f2ed8e951f3dbd8f048f12fb3aec2385a80 Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Sat, 24 Oct 2026 20:18:43 -0500 Subject: [PATCH 3/4] test(passthrough): cover x-litellm-upstream-model through spend rows and rollups Drives a config-defined pass-through against a scripted upstream and reads back the spend row, the daily team rollup and /team/daily/activity. Covers the header winning over a body model on buffered and streamed responses, failure rows, a model-only header keeping cost_per_request, the no-header and blank header cases, and repeated or long header values. Refs LIT-9910 --- .../spend/test_passthrough_upstream_model.py | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 tests/integration/spend/test_passthrough_upstream_model.py diff --git a/tests/integration/spend/test_passthrough_upstream_model.py b/tests/integration/spend/test_passthrough_upstream_model.py new file mode 100644 index 00000000000..1238c327bee --- /dev/null +++ b/tests/integration/spend/test_passthrough_upstream_model.py @@ -0,0 +1,228 @@ +import json +import uuid +from collections.abc import Callable, Mapping +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from typing import Final + +import pytest +from integration._support.client import Gateway, JsonValue, Scenario, eventually, object_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import TypeAdapter + +pytestmark: Final = pytest.mark.shares_worker_state + +_REPORTED_MODEL: Final = "support-summarizer" +_REPORTED_COST: Final = 0.0123 +_REPORTED_TOKENS: Final = 4210 +_USAGE_HEADERS: Final = { + "x-litellm-response-cost": str(_REPORTED_COST), + "x-litellm-total-tokens": str(_REPORTED_TOKENS), +} +_MODEL_HEADER: Final = {"x-litellm-upstream-model": _REPORTED_MODEL} + + +def _upstream( + marker: str, headers: Mapping[str, str], *, status: int = 200, stream: bool = False +) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST", request + assert json.loads(request.body)["ticket_id"] == marker, request + if stream: + return Reply( + status=status, + content_type="text/event-stream", + chunks=(f'data: {{"delta": "{marker}"}}\n\n'.encode(), b"data: [DONE]\n\n"), + headers=headers, + ) + return Reply(status=status, body=json.dumps({"summary": marker}).encode(), headers=headers) + + return respond + + +def _endpoint(gateway: Gateway, scenario: Scenario, marker: str, target: str, **fields: JsonValue) -> str: + path: Final = f"/support-ai-{marker}" + created: Final = gateway.post( + "/config/pass_through_endpoint", {"path": path, "target": target, "auth": True, **fields} + ) + endpoint_id: Final = object_value(TypeAdapter(list[JsonValue]).validate_python(created["endpoints"])[0])["id"] + scenario.cleanups.callback( + lambda: gateway.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": str(endpoint_id)}) + ) + return path + + +def _spend_row(key: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT model, spend, total_tokens, status FROM "LiteLLM_SpendLogs" WHERE api_key=%s AND call_type=%s', + (sha256(key.encode()).hexdigest(), "pass_through_endpoint"), + ), + lambda values: len(values) == 1, + seconds=70, + ) + return rows[0] + + +def _row_values(row: Mapping[str, JsonValue]) -> tuple[JsonValue, float, JsonValue, JsonValue]: + spend: Final = row["spend"] + assert isinstance(spend, (int, float)), row + return row["model"], float(spend), row["total_tokens"], row["status"] + + +def test_upstream_reported_model_names_the_spend_row_daily_rollup_and_usage_breakdown(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER})) as wire, gateway.scenario() as scenario: + path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize") + team: Final = scenario.team() + key: Final = scenario.key(team_id=team, allowed_passthrough_routes=[path]) + + response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key) + assert response.status_code == 200, response.text + assert response.json() == {"summary": marker}, response.text + assert response.headers["x-litellm-upstream-model"] == _REPORTED_MODEL, response.headers + + assert _row_values(_spend_row(key)) == ( + _REPORTED_MODEL, + pytest.approx(_REPORTED_COST), + _REPORTED_TOKENS, + "success", + ) + daily: Final = eventually( + lambda: read_rows('SELECT model, spend FROM "LiteLLM_DailyTeamSpend" WHERE team_id=%s', (team,)), + lambda values: len(values) == 1, + seconds=70, + ) + assert (daily[0]["model"], daily[0]["spend"]) == (_REPORTED_MODEL, pytest.approx(_REPORTED_COST)), daily + + today: Final = datetime.now(timezone.utc) + activity: Final = gateway.request( + "GET", + "/team/daily/activity", + params={ + "team_ids": team, + "start_date": (today - timedelta(days=1)).strftime("%Y-%m-%d"), + "end_date": (today + timedelta(days=1)).strftime("%Y-%m-%d"), + }, + ) + assert activity.status_code == 200, activity.text + results: Final = object_value(activity.json())["results"] + assert isinstance(results, list) and len(results) == 1, activity.text + models: Final = object_value(object_value(object_value(results[0])["breakdown"])["models"]) + assert set(models) == {_REPORTED_MODEL}, activity.text + assert object_value(object_value(models[_REPORTED_MODEL])["metrics"])["spend"] == pytest.approx( + _REPORTED_COST + ), activity.text + + +@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streamed"]) +def test_upstream_reported_model_wins_over_the_request_body_model(gateway: Gateway, stream: bool) -> None: + marker: Final = uuid.uuid4().hex + with ( + wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER}, stream=stream)) as wire, + gateway.scenario() as scenario, + ): + path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize") + key: Final = scenario.key(allowed_passthrough_routes=[path]) + + response: Final = gateway.request("POST", path, {"ticket_id": marker, "model": "gpt-4o"}, key=key) + assert response.status_code == 200, response.text + assert marker in response.text, response.text + + assert _row_values(_spend_row(key)) == ( + _REPORTED_MODEL, + pytest.approx(_REPORTED_COST), + _REPORTED_TOKENS, + "success", + ) + + +def test_upstream_error_row_carries_the_reported_model_and_cost(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with ( + wire_server(_upstream(marker, {**_USAGE_HEADERS, **_MODEL_HEADER}, status=500)) as wire, + gateway.scenario() as scenario, + ): + path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize") + key: Final = scenario.key(allowed_passthrough_routes=[path]) + + response: Final = gateway.request("POST", path, {"ticket_id": marker, "model": "gpt-4o"}, key=key) + assert response.status_code == 500, response.text + + assert _row_values(_spend_row(key)) == ( + _REPORTED_MODEL, + pytest.approx(_REPORTED_COST), + _REPORTED_TOKENS, + "failure", + ) + + +def test_model_header_alone_keeps_the_configured_cost_per_request(gateway: Gateway) -> None: + """The name is a label: an upstream that reports only a model does not own + the cost, so the endpoint's flat cost_per_request still applies.""" + marker: Final = uuid.uuid4().hex + with wire_server(_upstream(marker, _MODEL_HEADER)) as wire, gateway.scenario() as scenario: + path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize", cost_per_request=0.5) + key: Final = scenario.key(allowed_passthrough_routes=[path]) + + response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key) + assert response.status_code == 200, response.text + + model, spend, _, status = _row_values(_spend_row(key)) + assert (model, spend, status) == (_REPORTED_MODEL, pytest.approx(0.5), "success") + + +@pytest.mark.parametrize( + ("body_model", "header_value", "expected_model"), + [(None, None, "unknown"), ("gpt-4o", None, "gpt-4o"), ("gpt-4o", " ", "gpt-4o")], + ids=["no_header_no_body_model", "no_header_body_model", "blank_header_body_model"], +) +def test_without_a_usable_model_header_the_row_keeps_the_request_model( + gateway: Gateway, body_model: str | None, header_value: str | None, expected_model: str +) -> None: + marker: Final = uuid.uuid4().hex + headers: Final = {**_USAGE_HEADERS, **({} if header_value is None else {"x-litellm-upstream-model": header_value})} + body: Final[dict[str, JsonValue]] = {"ticket_id": marker, **({} if body_model is None else {"model": body_model})} + with wire_server(_upstream(marker, headers)) as wire, gateway.scenario() as scenario: + path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize") + key: Final = scenario.key(allowed_passthrough_routes=[path]) + + response: Final = gateway.request("POST", path, body, key=key) + assert response.status_code == 200, response.text + + assert _row_values(_spend_row(key)) == ( + expected_model, + pytest.approx(_REPORTED_COST), + _REPORTED_TOKENS, + "success", + ) + + +@pytest.mark.parametrize( + ("header_value", "expected_model"), + [ + (f"{_REPORTED_MODEL}, other-name", _REPORTED_MODEL), + ("m" * 5000, "m" * 256), + ], + ids=["repeated_header_keeps_first", "long_name_cut_to_256"], +) +@pytest.mark.parametrize("status", [200, 500], ids=["success", "failure"]) +def test_spend_row_keeps_a_bounded_first_model_name( + gateway: Gateway, header_value: str, expected_model: str, status: int +) -> None: + marker: Final = uuid.uuid4().hex + headers: Final = {**_USAGE_HEADERS, "x-litellm-upstream-model": header_value} + with wire_server(_upstream(marker, headers, status=status)) as wire, gateway.scenario() as scenario: + path: Final = _endpoint(gateway, scenario, marker, wire.url + "/v1/summarize") + key: Final = scenario.key(allowed_passthrough_routes=[path]) + + response: Final = gateway.request("POST", path, {"ticket_id": marker}, key=key) + assert response.status_code == status, response.text + + assert _row_values(_spend_row(key)) == ( + expected_model, + pytest.approx(_REPORTED_COST), + _REPORTED_TOKENS, + "success" if status == 200 else "failure", + ) From 619818e04e54cd0b7370d3fdd7daac74c9b63c24 Mon Sep 17 00:00:00 2001 From: Mrinal Chanshetty Date: Sat, 24 Oct 2026 20:58:08 -0500 Subject: [PATCH 4/4] test(passthrough): annotate new test locals with Final Refs LIT-9910 --- .../test_pass_through_endpoints.py | 4 ++-- .../test_upstream_usage_headers.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 9e30f0b5aca..74caba73e71 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -6866,7 +6866,7 @@ async def test_passthrough_records_model_reported_by_upstream(request_body: Mapp payloads, _ = await _run_upstream_reporting_passthrough(_UPSTREAM_SUMMARIZER_HEADERS, request_body=request_body) assert len(payloads) == 1 - logged = cast(types_utils.StandardLoggingPayload, payloads[0]) + logged: Final = cast(types_utils.StandardLoggingPayload, payloads[0]) assert (logged["model"], logged["response_cost"], logged["total_tokens"]) == ( "support-summarizer", 0.0123, @@ -6900,7 +6900,7 @@ async def test_passthrough_records_upstream_reported_model_on_error_response(): ) mock_proxy_logging.post_call_failure_hook.assert_awaited_once() - request_data = mock_proxy_logging.post_call_failure_hook.await_args.kwargs["request_data"] + request_data: Final = mock_proxy_logging.post_call_failure_hook.await_args.kwargs["request_data"] assert (request_data["model"], request_data["response_cost"], request_data["combined_usage_object"]) == ( "support-summarizer", 0.0123, diff --git a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py index f0b8ffd3664..9dd0233ad8b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py +++ b/tests/unit/proxy/pass_through_endpoints/test_upstream_usage_headers.py @@ -1,3 +1,5 @@ +from typing import Final + import httpx import pytest @@ -148,7 +150,9 @@ def test_parse_returns_no_model_when_upstream_sends_none(): def test_parse_uses_the_first_value_of_a_repeated_model_header(): - headers = httpx.Headers([("x-litellm-upstream-model", "support-summarizer"), ("x-litellm-upstream-model", "other")]) + headers: Final = httpx.Headers( + [("x-litellm-upstream-model", "support-summarizer"), ("x-litellm-upstream-model", "other")] + ) assert parse_upstream_reported_model(headers) == "support-summarizer" @@ -167,7 +171,7 @@ def test_parse_cuts_a_long_model_name_to_the_spend_log_limit(raw_model: str, exp def _logging_obj_for_model(model: str) -> LiteLLMLoggingObj: - logging_obj = _logging_obj() + logging_obj: Final = _logging_obj() logging_obj.update_environment_variables( model=model, user="unknown", optional_params={}, litellm_params={}, call_type="pass_through_endpoint" ) @@ -175,9 +179,9 @@ def _logging_obj_for_model(model: str) -> LiteLLMLoggingObj: def test_apply_model_replaces_the_request_model(): - logging_obj = _logging_obj_for_model("gpt-4o") + logging_obj: Final = _logging_obj_for_model("gpt-4o") - reported = apply_upstream_reported_model( + reported: Final = apply_upstream_reported_model( logging_obj=logging_obj, headers=_headers(**{"x-litellm-upstream-model": "support-summarizer"}), ) @@ -187,7 +191,7 @@ def test_apply_model_replaces_the_request_model(): def test_apply_model_leaves_request_model_alone_when_header_is_blank(): - logging_obj = _logging_obj_for_model("gpt-4o") + logging_obj: Final = _logging_obj_for_model("gpt-4o") assert ( apply_upstream_reported_model(logging_obj=logging_obj, headers=_headers(**{"x-litellm-upstream-model": " "})) @@ -199,8 +203,8 @@ def test_apply_model_leaves_request_model_alone_when_header_is_blank(): def test_model_header_alone_does_not_make_the_upstream_own_the_cost(): """The model name is a label: without a cost or token header the request must still fall back to cost_per_request, not be recorded at 0.""" - logging_obj = _logging_obj() - headers = _headers(**{"x-litellm-upstream-model": "support-summarizer"}) + logging_obj: Final = _logging_obj() + headers: Final = _headers(**{"x-litellm-upstream-model": "support-summarizer"}) apply_upstream_reported_usage(logging_obj=logging_obj, headers=headers) apply_upstream_reported_model(logging_obj=logging_obj, headers=headers)