diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 5e07632dcc8..1550e6d280c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1522,9 +1522,9 @@ def _adjusted_provider_billed_cost( ) _store_cost_breakdown_in_logging_obj( litellm_logging_obj=litellm_logging_obj, - prompt_tokens_cost_usd_dollar=None, - completion_tokens_cost_usd_dollar=None, - cost_for_built_in_tools_cost_usd_dollar=None, + prompt_tokens_cost_usd_dollar=0.0, + completion_tokens_cost_usd_dollar=adjusted_cost, + cost_for_built_in_tools_cost_usd_dollar=0.0, total_cost_usd_dollar=adjusted_cost, original_cost=reported_cost, discount_percent=discount_percent, diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index d940baf8b56..58ff7c2218b 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1017,11 +1017,20 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) -USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) +OWN_REPORTED_COST_READER_PROVIDERS: Final = frozenset({"xai", "perplexity", "opencode_go"}) + + +def _carries_litellm_bill(response: object) -> bool: + """A response the stream builder already priced carries litellm's own bill in ``usage.cost`` too.""" + hidden: Final[object] = getattr(response, "_hidden_params", None) + stamped: Final = hidden.get("response_cost") if isinstance(hidden, Mapping) else None + return isinstance(stamped, (int, float)) and not isinstance(stamped, bool) def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: - if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + """The charge an OpenAI-compatible upstream reports in ``usage.cost``, billed unless the provider has its own + reader or the number is litellm's own bill written back onto the usage.""" + if custom_llm_provider in OWN_REPORTED_COST_READER_PROVIDERS or _carries_litellm_bill(response): return None cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 7c91ffdff76..e308628fbdb 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import ( - USAGE_COST_BILLED_PROVIDERS, + OWN_REPORTED_COST_READER_PROVIDERS, chunk_usage, provider_reported_usage_cost, ) @@ -1356,9 +1356,6 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS - - def _logged_provider(logging_obj: Optional["Logging"]) -> object: if logging_obj is None: return None @@ -1366,10 +1363,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object: return provider -def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: - return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS - - def _stream_builder_response_cost( response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] ) -> float | None: @@ -1380,9 +1373,6 @@ def _stream_builder_response_cost( for chunk in reversed(chunks) ) return next((cost for cost in reported if cost is not None), None) - usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) - if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): - return float(usage_cost) if logging_obj is not None: return None provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor @@ -1429,7 +1419,9 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if isinstance(getattr(usage, "cost", None), (int, float)): + if _logged_provider(logging_obj) in OWN_REPORTED_COST_READER_PROVIDERS and isinstance( + getattr(usage, "cost", None), (int, float) + ): return computed_cost: Final = logging_obj.response_cost_calculator(result=response) if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 53fd5c1cec2..923ce6136aa 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -2,6 +2,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +from collections.abc import Mapping from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( @@ -11878,7 +11879,215 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS } ) + +# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill, +# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed +# replies alike. A deployment the operator priced keeps the operator's rates on both. +_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected( + spend=0.037, input_cost=0.023125, output_cost=0.013875, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06) +_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected( + spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_CHUNK: Final = { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1, + "model": "gateway-reported-cost-fixture", +} + +GATEWAY_REPORTED_COST: Final = CostTrackingTestCase( + name="gateway-reported-cost", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False}, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1, + "model": "gateway-reported-cost-fixture", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": _GATEWAY_REPORTED_USAGE, + }, + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase( + name="gateway-reported-cost-stream", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "reported cost"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE}, + done=True, + ), + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = { + "id": "msg_gateway", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], +} +_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gateway-reported-cost-fixture", + "output": [_GATEWAY_RESPONSES_ITEM], + "usage": _GATEWAY_RESPONSES_USAGE, +} + +GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase( + name="gateway-reported-cost-responses", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/responses", + request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}}, + response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-stream", + "request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True}, + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "response.created", + "sequence_number": 0, + "response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_gateway", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY}, + ), + ), + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + + +_GATEWAY_ZERO_USAGE: Final = {**_GATEWAY_REPORTED_USAGE, "cost": 0} + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced", + "response": JsonResponse( + content_type="application/json", + body={**GATEWAY_REPORTED_COST.response.body, "usage": _GATEWAY_ZERO_USAGE}, + ), + } +) + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced-stream", + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_ZERO_USAGE}, + done=True, + ), + ), + } +) + +_GATEWAY_OPENROUTER_OPERATOR_PRICED: Final = _GATEWAY_OPERATOR_PRICED.model_copy( + update={"model": "openrouter/vendor/gateway-reported-cost-fixture"} +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = ( + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced-stream", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } + ) +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( + GATEWAY_REPORTED_COST, + GATEWAY_REPORTED_COST_STREAM, + GATEWAY_REPORTED_COST_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_RESPONSES, + GATEWAY_REPORTED_COST_RESPONSES_STREAM, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12120,6 +12329,22 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( + StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM + ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM + ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, + ), + StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 32722b37c9b..da4c2b6e859 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_above_128k_tokens": 2.2e-07, "output_cost_per_token_above_128k_tokens": 8.8e-07, }, + "gateway-reported-cost-fixture": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + }, "openrouter/anthropic/claude-sonnet-5": { "litellm_provider": "openrouter", "mode": "chat",