From ab44c8a8ce76dd5a99df94851687fd6bc4547fa4 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 24 Jul 2026 18:26:12 -0700 Subject: [PATCH] fix(passthrough): let an upstream-reported cost outrank cost_per_request PassThroughGenericEndpoint.cost_per_request defaults to 0.0, so every config-defined endpoint forwards a flat 0.0 even when the operator never configured one, and the success handler applied it over whatever cost was already established. That silently zeroed the cost an upstream reported for the request. The flat value is an estimate for targets LiteLLM cannot price, so it now yields to a target that priced the request itself; it still applies unchanged when no cost was reported. --- .../pass_through_endpoints/success_handler.py | 9 +++++ .../upstream_usage_headers.py | 14 ++++++++ .../test_pass_through_endpoints.py | 33 ++++++++++++++++++- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 932216141fe..84a956f0c29 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -30,6 +30,7 @@ from .llm_provider_handlers.gemini_passthrough_logging_handler import ( from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) +from .upstream_usage_headers import upstream_reported_cost cohere_passthrough_logging_handler = CoherePassthroughLoggingHandler() @@ -448,10 +449,18 @@ class PassThroughEndpointLogging: Only set the cost per request if it's set in the passthrough logging payload. If it's not set, don't set it in the logging object. + + An upstream that priced the request itself always wins: ``cost_per_request`` + is a flat per-request estimate for targets LiteLLM cannot price, and it + defaults to 0.0 on every config-defined endpoint, so honoring it here + would zero out the real cost the upstream reported. """ ######################################################### # Check if cost per request is set ######################################################### + if upstream_reported_cost(logging_obj) is not None: + return kwargs + if passthrough_logging_payload.get("cost_per_request") is not None: kwargs["response_cost"] = passthrough_logging_payload.get("cost_per_request") logging_obj.model_call_details["response_cost"] = passthrough_logging_payload.get("cost_per_request") diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index 6531cff1c1d..e147dcb9547 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -18,6 +18,11 @@ from litellm.types.utils import Usage UPSTREAM_RESPONSE_COST_HEADER = "x-litellm-response-cost" UPSTREAM_TOTAL_TOKENS_HEADER = "x-litellm-total-tokens" +# model_call_details key holding what the upstream reported, so later stages of +# the success path can tell an upstream-reported cost apart from one LiteLLM +# derived itself. +UPSTREAM_REPORTED_USAGE_KEY = "_litellm_upstream_reported_usage" + @dataclass(frozen=True, slots=True) class UpstreamReportedUsage: @@ -113,8 +118,17 @@ def apply_upstream_reported_usage( reported = parse_upstream_reported_usage(headers) if reported is None: return None + logging_obj.model_call_details[UPSTREAM_REPORTED_USAGE_KEY] = reported if reported.response_cost is not None: logging_obj.model_call_details["response_cost"] = reported.response_cost if reported.total_tokens is not None: logging_obj.model_call_details["combined_usage_object"] = Usage(total_tokens=reported.total_tokens) return reported + + +def upstream_reported_cost(logging_obj: LiteLLMLoggingObj) -> float | None: + """The cost the upstream reported for this request, if it reported one.""" + reported = logging_obj.model_call_details.get(UPSTREAM_REPORTED_USAGE_KEY) + if not isinstance(reported, UpstreamReportedUsage): + return None + return reported.response_cost diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 10706b4ea64..60340cfa47b 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -4670,7 +4670,9 @@ def _enter_upstream_usage_mocks(stack, parsed_body): return mock_proxy_logging, enqueued -async def _run_upstream_reporting_passthrough(upstream_headers, status_code=200): +async def _run_upstream_reporting_passthrough( + upstream_headers, status_code=200, cost_per_request=None +): """Drive a generic pass-through against an upstream that reports its own cost/usage. Returns (recorded standard logging payloads, proxy logging mock).""" from litellm.proxy._types import UserAPIKeyAuth @@ -4694,6 +4696,7 @@ async def _run_upstream_reporting_passthrough(upstream_headers, status_code=200) user_api_key_dict=UserAPIKeyAuth( api_key="sk-upstream-usage", team_id="team-fil" ), + cost_per_request=cost_per_request, ) for coroutine in enqueued: await coroutine @@ -4829,3 +4832,31 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea assert len(recorder.payloads) == 1 assert recorder.payloads[0]["response_cost"] == 0.00312 assert recorder.payloads[0]["total_tokens"] == 4021 + + +@pytest.mark.asyncio +async def test_upstream_reported_cost_survives_default_cost_per_request(): + """ + PassThroughGenericEndpoint.cost_per_request defaults to 0.0, so every + config-defined endpoint forwards a 0.0 flat cost even when the operator + never configured one. That flat estimate must not overwrite the real cost + the upstream reported for the request. + """ + payloads, _ = await _run_upstream_reporting_passthrough( + { + "x-litellm-response-cost": "0.000415", + "x-litellm-total-tokens": "1874", + }, + cost_per_request=0.0, + ) + + assert len(payloads) == 1 + assert payloads[0]["response_cost"] == 0.000415 + + +@pytest.mark.asyncio +async def test_configured_cost_per_request_still_applies_without_usage_headers(): + payloads, _ = await _run_upstream_reporting_passthrough({}, cost_per_request=0.25) + + assert len(payloads) == 1 + assert payloads[0]["response_cost"] == 0.25