diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 932216141fe..84a956f0c29 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -30,6 +30,7 @@ from .llm_provider_handlers.gemini_passthrough_logging_handler import ( from .llm_provider_handlers.vertex_passthrough_logging_handler import ( VertexPassthroughLoggingHandler, ) +from .upstream_usage_headers import upstream_reported_cost cohere_passthrough_logging_handler = CoherePassthroughLoggingHandler() @@ -448,10 +449,18 @@ class PassThroughEndpointLogging: Only set the cost per request if it's set in the passthrough logging payload. If it's not set, don't set it in the logging object. + + An upstream that priced the request itself always wins: ``cost_per_request`` + is a flat per-request estimate for targets LiteLLM cannot price, and it + defaults to 0.0 on every config-defined endpoint, so honoring it here + would zero out the real cost the upstream reported. """ ######################################################### # Check if cost per request is set ######################################################### + if upstream_reported_cost(logging_obj) is not None: + return kwargs + if passthrough_logging_payload.get("cost_per_request") is not None: kwargs["response_cost"] = passthrough_logging_payload.get("cost_per_request") logging_obj.model_call_details["response_cost"] = passthrough_logging_payload.get("cost_per_request") diff --git a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py index 6531cff1c1d..e147dcb9547 100644 --- a/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py +++ b/litellm/proxy/pass_through_endpoints/upstream_usage_headers.py @@ -18,6 +18,11 @@ from litellm.types.utils import Usage UPSTREAM_RESPONSE_COST_HEADER = "x-litellm-response-cost" UPSTREAM_TOTAL_TOKENS_HEADER = "x-litellm-total-tokens" +# model_call_details key holding what the upstream reported, so later stages of +# the success path can tell an upstream-reported cost apart from one LiteLLM +# derived itself. +UPSTREAM_REPORTED_USAGE_KEY = "_litellm_upstream_reported_usage" + @dataclass(frozen=True, slots=True) class UpstreamReportedUsage: @@ -113,8 +118,17 @@ def apply_upstream_reported_usage( reported = parse_upstream_reported_usage(headers) if reported is None: return None + logging_obj.model_call_details[UPSTREAM_REPORTED_USAGE_KEY] = reported if reported.response_cost is not None: logging_obj.model_call_details["response_cost"] = reported.response_cost if reported.total_tokens is not None: logging_obj.model_call_details["combined_usage_object"] = Usage(total_tokens=reported.total_tokens) return reported + + +def upstream_reported_cost(logging_obj: LiteLLMLoggingObj) -> float | None: + """The cost the upstream reported for this request, if it reported one.""" + reported = logging_obj.model_call_details.get(UPSTREAM_REPORTED_USAGE_KEY) + if not isinstance(reported, UpstreamReportedUsage): + return None + return reported.response_cost diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 10706b4ea64..60340cfa47b 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -4670,7 +4670,9 @@ def _enter_upstream_usage_mocks(stack, parsed_body): return mock_proxy_logging, enqueued -async def _run_upstream_reporting_passthrough(upstream_headers, status_code=200): +async def _run_upstream_reporting_passthrough( + upstream_headers, status_code=200, cost_per_request=None +): """Drive a generic pass-through against an upstream that reports its own cost/usage. Returns (recorded standard logging payloads, proxy logging mock).""" from litellm.proxy._types import UserAPIKeyAuth @@ -4694,6 +4696,7 @@ async def _run_upstream_reporting_passthrough(upstream_headers, status_code=200) user_api_key_dict=UserAPIKeyAuth( api_key="sk-upstream-usage", team_id="team-fil" ), + cost_per_request=cost_per_request, ) for coroutine in enqueued: await coroutine @@ -4829,3 +4832,31 @@ async def test_streaming_passthrough_records_cost_and_tokens_reported_by_upstrea assert len(recorder.payloads) == 1 assert recorder.payloads[0]["response_cost"] == 0.00312 assert recorder.payloads[0]["total_tokens"] == 4021 + + +@pytest.mark.asyncio +async def test_upstream_reported_cost_survives_default_cost_per_request(): + """ + PassThroughGenericEndpoint.cost_per_request defaults to 0.0, so every + config-defined endpoint forwards a 0.0 flat cost even when the operator + never configured one. That flat estimate must not overwrite the real cost + the upstream reported for the request. + """ + payloads, _ = await _run_upstream_reporting_passthrough( + { + "x-litellm-response-cost": "0.000415", + "x-litellm-total-tokens": "1874", + }, + cost_per_request=0.0, + ) + + assert len(payloads) == 1 + assert payloads[0]["response_cost"] == 0.000415 + + +@pytest.mark.asyncio +async def test_configured_cost_per_request_still_applies_without_usage_headers(): + payloads, _ = await _run_upstream_reporting_passthrough({}, cost_per_request=0.25) + + assert len(payloads) == 1 + assert payloads[0]["response_cost"] == 0.25