From a20609a6f2419ad8577c3cd09b207798fc63bedc Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:37:04 +0000 Subject: [PATCH] fix(cost): show litellm's bill in usage.cost on chat non-stream and both Responses paths A priced deployment showed its bill only on chat streams. The metadata pass now stamps the bill onto ModelResponse and ResponsesAPIResponse usage, the Responses stream stamp overwrites the upstream's number too, all through one stamp_billable_usage_cost guard, and the integration suite asserts a JSON reply's usage.cost equals the spend row and gains /v1/responses twins of the reported-cost cases --- litellm/litellm_core_utils/hidden_params.py | 10 ++ .../llm_response_utils/response_metadata.py | 26 +++-- .../streaming_chunk_builder_utils.py | 11 +- litellm/responses/streaming_iterator.py | 7 +- .../cost_calculation/assertions.py | 16 +++ .../cost_calculation/cases/openai.py | 80 ++++++++++++++ .../unit/responses/test_streaming_iterator.py | 27 ++++- tests/unit/test_cost_calculator.py | 102 ++++++++++++++++++ 8 files changed, 257 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index ed493394f03..7281b138894 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -147,6 +147,16 @@ def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None: setattr(usage, "cost", cost) +def stamp_billable_usage_cost(obj: object, usage: object, cost: object) -> float | None: + """Stamp a priced bill, a $0 one included once the provider put a number there; an unpriceable reply keeps its usage.""" + if isinstance(cost, bool) or not isinstance(cost, (int, float)) or cost < 0: + return None + if cost == 0 and getattr(usage, "cost", None) is None: + return None + stamp_billed_usage_cost(obj, usage, float(cost)) + return float(cost) + + def provider_stated_usage_cost(obj: object, usage_cost: object) -> object: """What the provider itself put in usage.cost, looking past a bill litellm stamped there.""" hidden_params: Final = get_hidden_params(obj) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..4379b937394 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -8,13 +8,16 @@ from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers +from litellm.litellm_core_utils.hidden_params import stamp_billable_usage_cost from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( EmbeddingResponse, HiddenParams, ModelResponse, TranscriptionResponse, + Usage, ) @@ -159,16 +162,18 @@ class ResponseMetadata: ## ADD OTHER HIDDEN PARAMS model_info: Final = kwargs.get("model_info", {}) or {} model_id: Final = model_info.get("id", None) + response_cost: Final = logging_obj.response_cost_calculator( + result=self.result, + litellm_model_name=model, + router_model_id=model_id, + cache_hit=cache_hit, + ) + self._stamp_billed_usage_cost(response_cost) new_params: Final = { "litellm_call_id": getattr(logging_obj, "litellm_call_id", None), "api_base": get_api_base(model=model or "", optional_params=kwargs), "model_id": model_id, - "response_cost": logging_obj.response_cost_calculator( - result=self.result, - litellm_model_name=model, - router_model_id=model_id, - cache_hit=cache_hit, - ), + "response_cost": response_cost, "additional_headers": process_response_headers( self._get_additional_headers_from_hidden_params() or {}, preserve_litellm_internal_headers=True, @@ -179,6 +184,15 @@ class ResponseMetadata: } self._update_hidden_params(new_params) + def _stamp_billed_usage_cost(self, response_cost: object) -> None: + """A client reads what litellm billed from usage.cost, streamed or not, priced deployment or not.""" + if not isinstance(self.result, (ModelResponse, ResponsesAPIResponse)): + return + usage: Final = getattr(self.result, "usage", None) + if not isinstance(usage, (Usage, ResponseAPIUsage)): + return + stamp_billable_usage_cost(self.result, usage, response_cost) + def _update_hidden_params(self, new_params: Mapping[str, object]) -> None: """ Update hidden params - handles when self._hidden_params is a dict or HiddenParams object diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 4c983d6b2c5..3425871a408 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billed_usage_cost +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billable_usage_cost from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1413,13 +1413,10 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if not isinstance(computed_cost, (int, float)) or computed_cost < 0: + billed: Final = stamp_billable_usage_cost(response, usage, logging_obj.response_cost_calculator(result=response)) + if billed is None: return - if computed_cost == 0 and getattr(usage, "cost", None) is None: - return - stamp_billed_usage_cost(response, usage, computed_cost) - _set_hidden_response_cost(response, float(computed_cost)) + _set_hidden_response_cost(response, billed) def _field_of(value: object, name: str) -> object: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 3ef9c7bee4d..ca6bbf0046c 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,7 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base -from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billed_usage_cost +from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billable_usage_cost from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1758,14 +1758,11 @@ def stamp_responses_usage_cost( if usage_obj is None: return response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives - if isinstance(getattr(usage_obj, "cost", None), (int, float)): - return try: cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj) except Exception: return - if isinstance(cost, (int, float)) and cost > 0: - stamp_billed_usage_cost(response_obj, usage_obj, cost) + stamp_billable_usage_cost(response_obj, usage_obj, cost) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/integration/cost_calculation/assertions.py b/tests/integration/cost_calculation/assertions.py index 6e2af74d1a2..f37bb32e3b3 100644 --- a/tests/integration/cost_calculation/assertions.py +++ b/tests/integration/cost_calculation/assertions.py @@ -55,6 +55,18 @@ def assert_streamed_usage_cost(case_name: str, endpoint: str, response_text: str assert approx_equal(seen, spend), f"{case_name}: streamed usage.cost {seen} != spend {spend}" +def assert_json_usage_cost(case_name: str, endpoint: str, response: httpx.Response, spend: float) -> None: + """A JSON reply's usage.cost is what the row billed, the upstream's own figure included, so a client and the + spend log never disagree on a priced deployment.""" + if endpoint not in STREAMED_COST_ENDPOINTS: + return + usage: Final = JSON_OBJECT.validate_json(response.text).get("usage") + cost: Final = usage.get("cost") if isinstance(usage, dict) else None + if not isinstance(cost, (int, float)) or isinstance(cost, bool): + return + assert approx_equal(float(cost), spend), f"{case_name}: usage.cost {cost} != spend {spend}" + + def assert_breakdown( case_name: str, response_content_type: str, @@ -160,6 +172,8 @@ def assert_exact( assert_total_is_sum_of_components(row, breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, expected.spend) + if response is not None and response_content_type == "application/json": + assert_json_usage_cost(case_name, response.request.url.path, response, expected.spend) def assert_recount( @@ -201,3 +215,5 @@ def assert_recount( assert_total_is_sum_of_components(row, row.breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, row.spend) + if response is not None and response_content_type == "application/json": + assert_json_usage_cost(case_name, response.request.url.path, response, row.spend) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 5e18b8dec45..fb7a90a634a 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -2,6 +2,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +from collections.abc import Mapping from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( @@ -11956,11 +11957,85 @@ GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STRE } ) +_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = { + "id": "msg_gateway", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], +} +_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gateway-reported-cost-fixture", + "output": [_GATEWAY_RESPONSES_ITEM], + "usage": _GATEWAY_RESPONSES_USAGE, +} + +GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase( + name="gateway-reported-cost-responses", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/responses", + request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}}, + response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-stream", + "request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True}, + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "response.created", + "sequence_number": 0, + "response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_gateway", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY}, + ), + ), + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GATEWAY_REPORTED_COST, GATEWAY_REPORTED_COST_STREAM, GATEWAY_REPORTED_COST_OPERATOR_PRICED, GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_RESPONSES, + GATEWAY_REPORTED_COST_RESPONSES_STREAM, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12206,6 +12281,11 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM ), + StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 5716f46c210..8d624a186ad 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -851,17 +851,36 @@ def test_stamp_responses_usage_cost_stamps_computed_cost(): logging_obj.response_cost_calculator.assert_called_once_with(result=response) -def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): +def test_stamp_responses_usage_cost_puts_the_bill_over_the_provider_reported_cost(): + from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, provider_stated_usage_cost from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() setattr(response.usage, "cost", 0.5) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 0.00032 stamp_responses_usage_cost(response, logging_obj) - assert getattr(response.usage, "cost", None) == pytest.approx(0.5) - logging_obj.response_cost_calculator.assert_not_called() + assert getattr(response.usage, "cost", None) == pytest.approx(0.00032) + assert provider_stated_usage_cost(response, response.usage.cost) == pytest.approx(0.5) + assert response._hidden_params[USAGE_COST_BEFORE_LITELLM_STAMP] == pytest.approx(0.5) + + +def test_stamp_responses_usage_cost_stamps_a_zero_bill_only_over_a_provider_reported_cost(): + from litellm.responses.streaming_iterator import stamp_responses_usage_cost + + unpriced = _responses_api_response_with_usage() + reported = _responses_api_response_with_usage() + setattr(reported.usage, "cost", 0.5) + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 0.0 + + stamp_responses_usage_cost(unpriced, logging_obj) + stamp_responses_usage_cost(reported, logging_obj) + + assert getattr(unpriced.usage, "cost", None) is None + assert getattr(reported.usage, "cost", None) == 0.0 def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: @@ -889,13 +908,13 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): } ) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 3e-05 stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) assert response.usage.output_tokens_details.reasoning_tokens == 117 - logging_obj.response_cost_calculator.assert_not_called() def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index a52efc6b41e..83724d7a2fc 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -24,6 +24,8 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.llm_response_utils.response_metadata import update_response_metadata +from litellm.responses.streaming_iterator import stamp_responses_usage_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, ) @@ -10685,6 +10687,106 @@ def test_a_zero_bill_is_stamped_over_the_provider_reported_usage_cost_on_a_strea assert logging_obj.response_cost_calculator(result=response) == 0.0 +_PRICED_DEPLOYMENT_ID: Final = "reported-cost-deployment-priced-by-the-operator" +_PRICED_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType( + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 6e-6, + "metadata": {"model_info": {"id": _PRICED_DEPLOYMENT_ID}}, + } +) + + +def _reported_cost_usage() -> Usage: + return Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037) + + +def _priced_logging(stream: bool, call_type: str) -> Logging: + """The logging object the proxy hands a priced deployment's call, the deployment under metadata.model_info.""" + logging_obj: Final = _reported_cost_logging(stream, _PRICED_LITELLM_PARAMS) + logging_obj.call_type = call_type + return logging_obj + + +def _reported_cost_responses_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_reported", + created_at=1, + status="completed", + model="reported-cost-fixture", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=0.037), + ) + + +def _client_visible_reply(route: str) -> tuple[object, Logging]: + """The object a client of the given route reads usage.cost from, after litellm priced it.""" + match route: + case "chat": + logging_obj: Logging = _priced_logging(False, "completion") + response: object = ModelResponse( + id="chatcmpl-reported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=_reported_cost_usage(), + ) + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + case "chat-stream": + logging_obj = _priced_logging(True, "completion") + response = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, _reported_cost_usage()), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + case "responses": + logging_obj = _priced_logging(False, "aresponses") + response = _reported_cost_responses_response() + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + case _: + logging_obj = _priced_logging(True, "aresponses") + response = _reported_cost_responses_response() + stamp_responses_usage_cost(response, logging_obj) + return response, logging_obj + + +@pytest.mark.parametrize("route", ["chat", "chat-stream", "responses", "responses-stream"]) +def test_the_client_reads_what_litellm_billed_from_usage_cost_on_every_route( + monkeypatch: pytest.MonkeyPatch, route: str +) -> None: + """A priced deployment showed its bill on chat streams only; the other three routes still showed the upstream's number.""" + monkeypatch.setitem( + litellm.model_cost, + _PRICED_DEPLOYMENT_ID, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + ) + bill: Final = 100 * 2e-6 + 20 * 6e-6 + + response, logging_obj = _client_visible_reply(route) + + usage: Final = getattr(response, "usage", None) + assert usage is not None and usage.cost == pytest.approx(bill) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: