From 271fdbd22e6b2da4b81211909aa94faab7e8373f Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 26 Aug 2026 16:05:56 -0700 Subject: [PATCH] fix(proxy): keep the response-cost headers on calls priced at zero Pricing responses reads and vector-store management routes at zero dropped the whole x-litellm-response-cost family off those replies. The header build reads a falsy zero as a cost this response never recorded and filters it out, and a call that returns before pricing stores no cost breakdown for the component headers to read, so a client parsing the cost off a read got a KeyError where it had previously been handed a number. Those calls now advertise the family at zero. Retrieving a background response, and the cost poller's read of one, still report their real cost. The params-taking form of the predicate moves from opentelemetry into internal_call_metadata so the proxy header build and the OTEL recorders share one copy. --- litellm/integrations/opentelemetry.py | 32 +--- litellm/integrations/otel/plumbing/metrics.py | 4 +- .../internal_call_metadata.py | 20 +++ litellm/proxy/common_request_processing.py | 31 +++- .../proxy/test_common_request_processing.py | 143 ++++++++++++++++++ 5 files changed, 202 insertions(+), 28 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index cad9792c969..dcf9924b131 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, Final, TypedDict, cast import litellm from litellm._logging import verbose_logger -from litellm.constants import NON_INFERENCE_CALL_TYPES from litellm.integrations._types.open_inference import ( OpenInferenceSpanKindValues, SpanAttributes, @@ -23,7 +22,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import ( ) from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.semconv import Metric -from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call +from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.secret_redaction import redact_string from litellm.litellm_core_utils.service_tier_utils import ( @@ -240,25 +239,6 @@ def _freeze_for_dedupe(value: object, _depth: int = 0) -> HashableScope: return repr(value) -def _is_unbilled_non_inference( - call_type: str | None, litellm_params: Mapping[str, object] | None, response_obj: object -) -> bool: - """Whether this call is a read or management route whose token counts describe an - earlier request rather than this one. - - The call-type membership test runs first so that inference traffic, which is every - request in a normal workload, never pays for the metadata merge behind it. - """ - if call_type not in NON_INFERENCE_CALL_TYPES: - return False - from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup - - metadata: Final = ( - StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) if litellm_params is not None else None - ) - return is_unbilled_non_inference_call(call_type, metadata, response_obj) - - def _shutdown_tracer_provider(provider: "_SDKTracerProvider") -> None: """Flush and stop a dropped provider so its exporter thread is reclaimed.""" try: @@ -1667,7 +1647,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if ( self._token_usage_histogram and response_obj - and not _is_unbilled_non_inference(kwargs.get("call_type"), params, response_obj) + and not is_unbilled_non_inference_call_from_params(kwargs.get("call_type"), params, response_obj) and (usage := response_obj.get("usage")) ): in_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"} @@ -1745,7 +1725,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if not self._time_per_output_token_histogram: return - if _is_unbilled_non_inference(kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj): + if is_unbilled_non_inference_call_from_params( + kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj + ): return # Get completion tokens from response_obj @@ -2500,7 +2482,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): usage: Final = ( response_obj.get("usage") if response_obj - and not _is_unbilled_non_inference(kwargs.get("call_type"), litellm_params, response_obj) + and not is_unbilled_non_inference_call_from_params( + kwargs.get("call_type"), litellm_params, response_obj + ) else None ) if usage: diff --git a/litellm/integrations/otel/plumbing/metrics.py b/litellm/integrations/otel/plumbing/metrics.py index 69116ebe20c..c7e491c002a 100644 --- a/litellm/integrations/otel/plumbing/metrics.py +++ b/litellm/integrations/otel/plumbing/metrics.py @@ -21,7 +21,6 @@ from litellm.integrations.opentelemetry import ( METRIC_METADATA_KEYS, TOKEN_TYPE_ATTRIBUTE, _build_metric_attribute_filter, - _is_unbilled_non_inference, _resolve_metric_attribute_filter, ) from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds @@ -33,6 +32,7 @@ from litellm.integrations.otel.model.semconv import ( resolve_provider, ) from litellm.integrations.otel.model.utils import to_seconds +from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -199,7 +199,7 @@ class GenAIMetricRecorder: ) -> None: common_attrs: Final = self._filter_attributes(self._bounded_attributes(kwargs)) duration_s: Final = (end_time - start_time).total_seconds() - usage_is_replayed: Final = _is_unbilled_non_inference( + usage_is_replayed: Final = is_unbilled_non_inference_call_from_params( kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj ) diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index 05c857f8c89..34d5797a6d8 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -79,6 +79,26 @@ def is_unbilled_non_inference_call( return metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN +def is_unbilled_non_inference_call_from_params( + call_type: str | None, + litellm_params: Mapping[str, object] | None, + response: object, +) -> bool: + """:func:`is_unbilled_non_inference_call` for callers holding raw ``litellm_params``. + + The call-type membership test runs first so that inference traffic, which is every + request in a normal workload, never pays for the metadata merge behind it. + """ + if call_type not in NON_INFERENCE_CALL_TYPES: + return False + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + + metadata: Final = ( + StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params) if litellm_params is not None else None + ) + return is_unbilled_non_inference_call(call_type, metadata, response) + + def sanitize_user_api_key_auth(auth: object) -> object: """Copy of the auth object with its budget reservation removed; the cost callback falls back to reading the reservation from inside the auth object.""" diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index dbbf9cb673e..0699f28cf75 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -27,6 +27,7 @@ from litellm.constants import ( LITELLM_DETAILED_TIMING, LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, + NON_INFERENCE_CALL_TYPES, RETURN_RAW_MODEL_NAME_METADATA_KEY, ROUTER_MODEL_NAME_RESPONSE_FIELD, STREAM_SSE_DATA_PREFIX, @@ -38,6 +39,7 @@ from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) +from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost from litellm.litellm_core_utils.llm_response_utils.get_headers import ( @@ -1293,15 +1295,35 @@ def _uncached_input_cost( return input_cost - (cache_read_cost or 0.0) - (cache_creation_cost or 0.0) +_ZERO_COST_BREAKDOWN: Final = CostBreakdownHeaderValues( + original_cost=0.0, + discount_amount=0.0, + margin_total_amount=0.0, + margin_percent=0.0, + input_cost=0.0, + output_cost=0.0, + tool_usage_cost=0.0, +) +"""The component split a call priced at zero advertises, so a client reading the cost headers off a +read or management route still finds the whole family rather than a partially populated one.""" + + def _get_cost_breakdown_from_logging_obj( litellm_logging_obj: LiteLLMLoggingObj | None, ) -> CostBreakdownHeaderValues: - """Extract discount, margin, and per-component cost information from logging object's cost breakdown.""" + """Extract discount, margin, and per-component cost information from logging object's cost breakdown. + + A non-inference call that priced at zero never records a breakdown, so its components are + reported as zero here. Any such call that did price normally (retrieving a background response, + and the cost poller's read of one) has a stored breakdown and takes the branch below instead. + """ if not litellm_logging_obj or not hasattr(litellm_logging_obj, "cost_breakdown"): return CostBreakdownHeaderValues() cost_breakdown: Final = litellm_logging_obj.cost_breakdown if not cost_breakdown: + if litellm_logging_obj.call_type in NON_INFERENCE_CALL_TYPES: + return _ZERO_COST_BREAKDOWN return CostBreakdownHeaderValues() return CostBreakdownHeaderValues( @@ -2577,11 +2599,16 @@ class ProxyBaseLLMRequestProcessing: additional_headers = hidden_params.get("additional_headers", {}) or {} recover_response_cost: Final = not response_cost and hidden_params.get("response_cost") is None - llm_cost_for_headers: Final = ( + computed_cost_for_headers: Final = ( self._response_cost_from_logging_obj(response=response, logging_obj=logging_obj) or "" if recover_response_cost else response_cost ) + llm_cost_for_headers: Final = ( + 0.0 + if is_unbilled_non_inference_call_from_params(logging_obj.call_type, logging_obj.litellm_params, response) + else computed_cost_for_headers + ) _, request_metadata_bucket = get_or_create_metadata_bucket(self.data) guardrail_cost_for_headers: Final = guardrail_information_cost( request_metadata_bucket.get("standard_logging_guardrail_information") diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 58714a5e319..c42c4d29ed7 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -30,6 +30,7 @@ from litellm.proxy.common_request_processing import ( _ClientDisconnectedBeforeFirstChunk, _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, + CostBreakdownHeaderValues, _has_attribute_error_in_chain, _is_azure_model_router_request, open_sse_before_first_byte, @@ -4974,6 +4975,148 @@ class TestResponseCostHeaderForTypedDictResponses: assert fastapi_response.headers["x-litellm-response-cost"] == "0.00123" +class TestCostHeadersForCallsPricedAtZero: + """ + Regression for LIT-5602. Pricing responses reads and vector-store management routes at + zero dropped the entire x-litellm-response-cost family off those replies: the header + build reads a falsy zero as "this response never recorded a cost" and filters it out, + and a call that returns before pricing stores no cost breakdown for the component + headers to read. A client parsing the cost off a read got a KeyError where it had + previously been handed a number. Those calls now advertise the whole family at zero. + """ + + @staticmethod + def _responses_read(*, background=False): + from litellm.types.llms.openai import ResponsesAPIResponse + + return ResponsesAPIResponse( + id="resp_lit5602", + created_at=0, + model="gpt-4.1-mini", + object="response", + output=[], + status="completed", + background=background, + usage={"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + ) + + @staticmethod + def _logging_obj(*, call_type, recovered_cost=0.0): + logging_obj = MagicMock() + logging_obj.litellm_call_id = "call-lit5602" + logging_obj.call_type = call_type + logging_obj.litellm_params = {} + logging_obj.cost_breakdown = None + logging_obj.model_call_details = {"response_cost": recovered_cost} + logging_obj._response_cost_calculator = MagicMock(return_value=recovered_cost) + logging_obj._enqueue_deferred_logging = None + logging_obj._on_deferred_stream_complete = None + return logging_obj + + async def _drive(self, *, monkeypatch, response, logging_obj, route_type): + import litellm.proxy.common_request_processing as crp + from litellm.proxy._types import UserAPIKeyAuth as RealUserAPIKeyAuth + + async def fake_route_request(**kwargs): + async def _llm_call(): + return response + + return _llm_call() + + monkeypatch.setattr(crp, "route_request", fake_route_request) + + async def fake_post_call_success_hook(data, user_api_key_dict, response): + return response + + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + proxy_logging_obj.post_call_success_hook = fake_post_call_success_hook + + fastapi_response = Response() + processing_obj = ProxyBaseLLMRequestProcessing(data={"litellm_logging_obj": logging_obj}) + + with patch.object( + ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails", return_value=False + ): + await processing_obj.base_process_llm_request( + request=MagicMock(spec=Request, headers={}), + fastapi_response=fastapi_response, + user_api_key_dict=RealUserAPIKeyAuth(api_key="sk-test"), + route_type=route_type, + proxy_logging_obj=proxy_logging_obj, + general_settings={}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=None, + llm_router=None, + skip_pre_call_logic=True, + ) + return fastapi_response + + @pytest.mark.asyncio + async def test_responses_read_emits_the_cost_header_family_at_zero(self, monkeypatch): + fastapi_response = await self._drive( + monkeypatch=monkeypatch, + response=self._responses_read(), + logging_obj=self._logging_obj(call_type="aget_responses"), + route_type="aget_responses", + ) + + assert fastapi_response.headers["x-litellm-response-cost"] == "0.0" + for component in ( + "original", + "discount-amount", + "margin-amount", + "margin-percent", + "input", + "output", + "tool-usage", + ): + assert fastapi_response.headers[f"x-litellm-response-cost-{component}"] == "0.0" + + @pytest.mark.asyncio + async def test_reading_a_background_response_keeps_its_real_cost(self, monkeypatch): + fastapi_response = await self._drive( + monkeypatch=monkeypatch, + response=self._responses_read(background=True), + logging_obj=self._logging_obj(call_type="aget_responses", recovered_cost=0.00042), + route_type="aget_responses", + ) + + assert float(fastapi_response.headers["x-litellm-response-cost"]) == pytest.approx(0.00042) + + @pytest.mark.asyncio + async def test_an_inference_call_without_a_recorded_cost_still_omits_the_header(self, monkeypatch): + """A chat completion has no zero-priced route, so a falsy cost there means the cost was + never recorded and the header stays absent rather than advertising a made-up zero.""" + fastapi_response = await self._drive( + monkeypatch=monkeypatch, + response=SimpleNamespace(_hidden_params={}), + logging_obj=self._logging_obj(call_type="acompletion"), + route_type="acompletion", + ) + + assert "x-litellm-response-cost" not in fastapi_response.headers + + def test_cost_breakdown_reports_zero_components_for_a_call_priced_at_zero(self): + breakdown = _get_cost_breakdown_from_logging_obj( + litellm_logging_obj=self._logging_obj(call_type="aget_responses") + ) + + assert breakdown.original_cost == 0.0 + assert breakdown.input_cost == 0.0 + assert breakdown.output_cost == 0.0 + assert breakdown.tool_usage_cost == 0.0 + + def test_cost_breakdown_stays_empty_for_an_inference_call(self): + breakdown = _get_cost_breakdown_from_logging_obj( + litellm_logging_obj=self._logging_obj(call_type="acompletion") + ) + + assert breakdown == CostBreakdownHeaderValues() + + class TestPreCallWithFallbacksOnLocalRateLimit: @pytest.mark.asyncio