diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index ed493394f03..7281b138894 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -147,6 +147,16 @@ def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None: setattr(usage, "cost", cost) +def stamp_billable_usage_cost(obj: object, usage: object, cost: object) -> float | None: + """Stamp a priced bill, a $0 one included once the provider put a number there; an unpriceable reply keeps its usage.""" + if isinstance(cost, bool) or not isinstance(cost, (int, float)) or cost < 0: + return None + if cost == 0 and getattr(usage, "cost", None) is None: + return None + stamp_billed_usage_cost(obj, usage, float(cost)) + return float(cost) + + def provider_stated_usage_cost(obj: object, usage_cost: object) -> object: """What the provider itself put in usage.cost, looking past a bill litellm stamped there.""" hidden_params: Final = get_hidden_params(obj) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..4379b937394 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -8,13 +8,16 @@ from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers +from litellm.litellm_core_utils.hidden_params import stamp_billable_usage_cost from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( EmbeddingResponse, HiddenParams, ModelResponse, TranscriptionResponse, + Usage, ) @@ -159,16 +162,18 @@ class ResponseMetadata: ## ADD OTHER HIDDEN PARAMS model_info: Final = kwargs.get("model_info", {}) or {} model_id: Final = model_info.get("id", None) + response_cost: Final = logging_obj.response_cost_calculator( + result=self.result, + litellm_model_name=model, + router_model_id=model_id, + cache_hit=cache_hit, + ) + self._stamp_billed_usage_cost(response_cost) new_params: Final = { "litellm_call_id": getattr(logging_obj, "litellm_call_id", None), "api_base": get_api_base(model=model or "", optional_params=kwargs), "model_id": model_id, - "response_cost": logging_obj.response_cost_calculator( - result=self.result, - litellm_model_name=model, - router_model_id=model_id, - cache_hit=cache_hit, - ), + "response_cost": response_cost, "additional_headers": process_response_headers( self._get_additional_headers_from_hidden_params() or {}, preserve_litellm_internal_headers=True, @@ -179,6 +184,15 @@ class ResponseMetadata: } self._update_hidden_params(new_params) + def _stamp_billed_usage_cost(self, response_cost: object) -> None: + """A client reads what litellm billed from usage.cost, streamed or not, priced deployment or not.""" + if not isinstance(self.result, (ModelResponse, ResponsesAPIResponse)): + return + usage: Final = getattr(self.result, "usage", None) + if not isinstance(usage, (Usage, ResponseAPIUsage)): + return + stamp_billable_usage_cost(self.result, usage, response_cost) + def _update_hidden_params(self, new_params: Mapping[str, object]) -> None: """ Update hidden params - handles when self._hidden_params is a dict or HiddenParams object diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 4c983d6b2c5..3425871a408 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billed_usage_cost +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billable_usage_cost from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1413,13 +1413,10 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if not isinstance(computed_cost, (int, float)) or computed_cost < 0: + billed: Final = stamp_billable_usage_cost(response, usage, logging_obj.response_cost_calculator(result=response)) + if billed is None: return - if computed_cost == 0 and getattr(usage, "cost", None) is None: - return - stamp_billed_usage_cost(response, usage, computed_cost) - _set_hidden_response_cost(response, float(computed_cost)) + _set_hidden_response_cost(response, billed) def _field_of(value: object, name: str) -> object: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 3ef9c7bee4d..ca6bbf0046c 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,7 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base -from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billed_usage_cost +from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billable_usage_cost from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1758,14 +1758,11 @@ def stamp_responses_usage_cost( if usage_obj is None: return response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives - if isinstance(getattr(usage_obj, "cost", None), (int, float)): - return try: cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj) except Exception: return - if isinstance(cost, (int, float)) and cost > 0: - stamp_billed_usage_cost(response_obj, usage_obj, cost) + stamp_billable_usage_cost(response_obj, usage_obj, cost) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/integration/cost_calculation/assertions.py b/tests/integration/cost_calculation/assertions.py index 6e2af74d1a2..f37bb32e3b3 100644 --- a/tests/integration/cost_calculation/assertions.py +++ b/tests/integration/cost_calculation/assertions.py @@ -55,6 +55,18 @@ def assert_streamed_usage_cost(case_name: str, endpoint: str, response_text: str assert approx_equal(seen, spend), f"{case_name}: streamed usage.cost {seen} != spend {spend}" +def assert_json_usage_cost(case_name: str, endpoint: str, response: httpx.Response, spend: float) -> None: + """A JSON reply's usage.cost is what the row billed, the upstream's own figure included, so a client and the + spend log never disagree on a priced deployment.""" + if endpoint not in STREAMED_COST_ENDPOINTS: + return + usage: Final = JSON_OBJECT.validate_json(response.text).get("usage") + cost: Final = usage.get("cost") if isinstance(usage, dict) else None + if not isinstance(cost, (int, float)) or isinstance(cost, bool): + return + assert approx_equal(float(cost), spend), f"{case_name}: usage.cost {cost} != spend {spend}" + + def assert_breakdown( case_name: str, response_content_type: str, @@ -160,6 +172,8 @@ def assert_exact( assert_total_is_sum_of_components(row, breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, expected.spend) + if response is not None and response_content_type == "application/json": + assert_json_usage_cost(case_name, response.request.url.path, response, expected.spend) def assert_recount( @@ -201,3 +215,5 @@ def assert_recount( assert_total_is_sum_of_components(row, row.breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, row.spend) + if response is not None and response_content_type == "application/json": + assert_json_usage_cost(case_name, response.request.url.path, response, row.spend) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 5e18b8dec45..fb7a90a634a 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -2,6 +2,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +from collections.abc import Mapping from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( @@ -11956,11 +11957,85 @@ GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STRE } ) +_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = { + "id": "msg_gateway", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], +} +_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gateway-reported-cost-fixture", + "output": [_GATEWAY_RESPONSES_ITEM], + "usage": _GATEWAY_RESPONSES_USAGE, +} + +GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase( + name="gateway-reported-cost-responses", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/responses", + request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}}, + response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-stream", + "request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True}, + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "response.created", + "sequence_number": 0, + "response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_gateway", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY}, + ), + ), + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GATEWAY_REPORTED_COST, GATEWAY_REPORTED_COST_STREAM, GATEWAY_REPORTED_COST_OPERATOR_PRICED, GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_RESPONSES, + GATEWAY_REPORTED_COST_RESPONSES_STREAM, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12206,6 +12281,11 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM ), + StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 5716f46c210..8d624a186ad 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -851,17 +851,36 @@ def test_stamp_responses_usage_cost_stamps_computed_cost(): logging_obj.response_cost_calculator.assert_called_once_with(result=response) -def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): +def test_stamp_responses_usage_cost_puts_the_bill_over_the_provider_reported_cost(): + from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, provider_stated_usage_cost from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() setattr(response.usage, "cost", 0.5) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 0.00032 stamp_responses_usage_cost(response, logging_obj) - assert getattr(response.usage, "cost", None) == pytest.approx(0.5) - logging_obj.response_cost_calculator.assert_not_called() + assert getattr(response.usage, "cost", None) == pytest.approx(0.00032) + assert provider_stated_usage_cost(response, response.usage.cost) == pytest.approx(0.5) + assert response._hidden_params[USAGE_COST_BEFORE_LITELLM_STAMP] == pytest.approx(0.5) + + +def test_stamp_responses_usage_cost_stamps_a_zero_bill_only_over_a_provider_reported_cost(): + from litellm.responses.streaming_iterator import stamp_responses_usage_cost + + unpriced = _responses_api_response_with_usage() + reported = _responses_api_response_with_usage() + setattr(reported.usage, "cost", 0.5) + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 0.0 + + stamp_responses_usage_cost(unpriced, logging_obj) + stamp_responses_usage_cost(reported, logging_obj) + + assert getattr(unpriced.usage, "cost", None) is None + assert getattr(reported.usage, "cost", None) == 0.0 def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: @@ -889,13 +908,13 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): } ) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 3e-05 stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) assert response.usage.output_tokens_details.reasoning_tokens == 117 - logging_obj.response_cost_calculator.assert_not_called() def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index a52efc6b41e..83724d7a2fc 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -24,6 +24,8 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.llm_response_utils.response_metadata import update_response_metadata +from litellm.responses.streaming_iterator import stamp_responses_usage_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, ) @@ -10685,6 +10687,106 @@ def test_a_zero_bill_is_stamped_over_the_provider_reported_usage_cost_on_a_strea assert logging_obj.response_cost_calculator(result=response) == 0.0 +_PRICED_DEPLOYMENT_ID: Final = "reported-cost-deployment-priced-by-the-operator" +_PRICED_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType( + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 6e-6, + "metadata": {"model_info": {"id": _PRICED_DEPLOYMENT_ID}}, + } +) + + +def _reported_cost_usage() -> Usage: + return Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037) + + +def _priced_logging(stream: bool, call_type: str) -> Logging: + """The logging object the proxy hands a priced deployment's call, the deployment under metadata.model_info.""" + logging_obj: Final = _reported_cost_logging(stream, _PRICED_LITELLM_PARAMS) + logging_obj.call_type = call_type + return logging_obj + + +def _reported_cost_responses_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_reported", + created_at=1, + status="completed", + model="reported-cost-fixture", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=0.037), + ) + + +def _client_visible_reply(route: str) -> tuple[object, Logging]: + """The object a client of the given route reads usage.cost from, after litellm priced it.""" + match route: + case "chat": + logging_obj: Logging = _priced_logging(False, "completion") + response: object = ModelResponse( + id="chatcmpl-reported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=_reported_cost_usage(), + ) + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + case "chat-stream": + logging_obj = _priced_logging(True, "completion") + response = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, _reported_cost_usage()), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + case "responses": + logging_obj = _priced_logging(False, "aresponses") + response = _reported_cost_responses_response() + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + case _: + logging_obj = _priced_logging(True, "aresponses") + response = _reported_cost_responses_response() + stamp_responses_usage_cost(response, logging_obj) + return response, logging_obj + + +@pytest.mark.parametrize("route", ["chat", "chat-stream", "responses", "responses-stream"]) +def test_the_client_reads_what_litellm_billed_from_usage_cost_on_every_route( + monkeypatch: pytest.MonkeyPatch, route: str +) -> None: + """A priced deployment showed its bill on chat streams only; the other three routes still showed the upstream's number.""" + monkeypatch.setitem( + litellm.model_cost, + _PRICED_DEPLOYMENT_ID, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + ) + bill: Final = 100 * 2e-6 + 20 * 6e-6 + + response, logging_obj = _client_visible_reply(route) + + usage: Final = getattr(response, "usage", None) + assert usage is not None and usage.cost == pytest.approx(bill) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: