From c200c2f9f8cb393d54136404b657016996622d90 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:09:17 +0000 Subject: [PATCH 01/26] fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced cost_per_token bills a finite non-negative usage.cost from any provider, split across input and output by the ratio of the token math, unless the caller priced the call itself. completion_cost drops the reported cost when the deployment has custom pricing or a base_model, so operator rates win on streams too, where stream_chunk_builder used to stamp usage.cost as the final response_cost ahead of custom pricing and margins. The xAI and Perplexity calculators lose their private copies of the same rule. Co-authored-by: AoHanBei Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 102 ++++++++++- .../streaming_chunk_builder_utils.py | 10 -- litellm/llms/perplexity/cost_calculator.py | 16 -- litellm/llms/xai/cost_calculator.py | 18 +- .../cost_calculation/cases/openai.py | 86 +++++++++ .../integration/cost_calculation/cost_map.py | 6 + .../test_streaming_handler.py | 8 +- .../test_perplexity_cost_calculator.py | 20 ++- .../unit/llms/xai/test_xai_cost_calculator.py | 10 +- tests/unit/test_cost_calculator.py | 166 ++++++++++++++++++ 10 files changed, 381 insertions(+), 61 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8331c68fe24..6ec8d00e37f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -5,7 +5,7 @@ import math import time from collections.abc import Mapping, Sequence from datetime import datetime -from functools import lru_cache +from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast @@ -22,7 +22,7 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, provider_reported_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -408,6 +408,19 @@ def _per_second_pricing_cost( return resolved_cost_per_second * seconds, 0.0 +def _billable_reported_cost(usage_object: Usage | None) -> float | None: + return _billable_header_cost(provider_reported_usage_cost(usage_object, None)) + + +def _split_reported_cost(reported_cost: float, priced: tuple[float, float] | None) -> tuple[float, float]: + """Providers report one total, so attribute it to input and output in the ratio the token math gives.""" + priced_total: Final = (priced[0] + priced[1]) if priced is not None else 0.0 + if priced is None or priced_total <= 0: + return 0.0, reported_cost + input_cost: Final = reported_cost * priced[0] / priced_total + return input_cost, reported_cost - input_cost + + def cost_per_token( model: str = "", prompt_tokens: int = 0, @@ -444,6 +457,87 @@ def cost_per_token( request_model: str | None = None, # original request model for router detection custom_model_info: OCRPricing | None = None, bedrock_model_info: ModelInfo | None = None, +) -> tuple[float, float]: + """Cost in USD for prompt and completion tokens. + + A numeric ``usage.cost`` the provider reported is the bill, split by the ratio of our own token math, + unless the caller priced the call itself with ``custom_cost_per_token`` or ``custom_cost_per_second``. + Without a reported cost the token math is the bill. + """ + priced_by_caller: Final = custom_cost_per_token is not None or custom_cost_per_second is not None + reported_cost: Final = None if priced_by_caller else _billable_reported_cost(usage_object) + rated: Final = partial( + _cost_per_token_from_rates, + model=model, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + response_time_ms=response_time_ms, + custom_llm_provider=custom_llm_provider, + region_name=region_name, + prompt_characters=prompt_characters, + completion_characters=completion_characters, + cache_creation_input_tokens=cache_creation_input_tokens, + cache_read_input_tokens=cache_read_input_tokens, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, + number_of_queries=number_of_queries, + usage_object=usage_object, + rerank_billed_units=rerank_billed_units, + call_type=call_type, + audio_transcription_file_duration=audio_transcription_file_duration, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + response=response, + request_model=request_model, + custom_model_info=custom_model_info, + bedrock_model_info=bedrock_model_info, + ) + if reported_cost is None: + return rated() + try: + return _split_reported_cost(reported_cost, rated()) + except Exception: # noqa: BLE001 # an unmapped model has no rates to split by, the reported total still bills + verbose_logger.debug("No rates for model=%s, billing the provider reported cost %s", model, reported_cost) + return _split_reported_cost(reported_cost, None) + + +def _cost_per_token_from_rates( + model: str = "", + prompt_tokens: int = 0, + completion_tokens: int = 0, + response_time_ms: float | None = 0.0, + custom_llm_provider: str | None = None, + region_name=None, + ### CHARACTER PRICING ### + prompt_characters: int | None = None, + completion_characters: int | None = None, + ### PROMPT CACHING PRICING ### - used for anthropic + cache_creation_input_tokens: int | None = 0, + cache_read_input_tokens: int | None = 0, + ### CUSTOM PRICING ### + custom_cost_per_token: CostPerToken | None = None, + custom_cost_per_second: float | None = None, + ### NUMBER OF QUERIES ### + number_of_queries: int | None = None, + ### USAGE OBJECT ### + usage_object: Usage | None = None, # just read the usage object if provided + ### BILLED UNITS ### + rerank_billed_units: RerankBilledUnits | None = None, + ### CALL TYPE ### + call_type: CallTypesLiteral = "completion", + audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds + ### SERVICE TIER ### + service_tier: str | None = None, # for OpenAI service tier pricing + ### DATA RESIDENCY ### + data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") + response: object | None = None, + ### REQUEST MODEL ### + request_model: str | None = None, # original request model for router detection + custom_model_info: OCRPricing | None = None, + bedrock_model_info: ModelInfo | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -1610,7 +1704,9 @@ def completion_cost( audio_transcription_file_duration: float = 0.0 provider_usage_object: Final = get_usage_object(completion_response=completion_response) cost_per_token_usage_object: Final[Usage | None] = ( - _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object + _without_provider_stated_cost(provider_usage_object) + if custom_pricing or base_model is not None + else provider_usage_object ) rerank_billed_units: RerankBilledUnits | None = None diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 68cdc204af4..4cdb9004619 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1350,9 +1350,6 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) - - def _logged_provider(logging_obj: Optional["Logging"]) -> object: if logging_obj is None: return None @@ -1360,10 +1357,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object: return provider -def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: - return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS - - def _stream_builder_response_cost( response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] ) -> float | None: @@ -1374,9 +1367,6 @@ def _stream_builder_response_cost( for chunk in reversed(chunks) ) return next((cost for cost in reported if cost is not None), None) - usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) - if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): - return float(usage_cost) if logging_obj is not None: return None provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 2d762adf958..a6825eacfad 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -24,22 +24,6 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - ## USE PRE-CALCULATED COST FROM PERPLEXITY IF AVAILABLE - ## Perplexity returns accurate cost in usage.cost.total_cost including request fees. - ## By the time it reaches here, ResponseAPIUsage.parse_cost has already flattened - ## that dict down to a float, so both shapes must be accepted. - cost_info: Final = getattr(usage, "cost", None) - total_cost: float | None = None - if isinstance(cost_info, dict): - total_cost = cost_info.get("total_cost") - elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool): - total_cost = float(cost_info) - if total_cost is not None: - # Return total cost as completion_cost (prompt_cost=0) since Perplexity - # doesn't break down by input/output in their cost object - return (0.0, float(total_cost)) - - ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it ## GET MODEL INFO model_info: Final = get_model_info(model=model, custom_llm_provider="perplexity") diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index aec535c2086..b62ed39c4fa 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -1,8 +1,8 @@ """ Helper util for handling XAI-specific cost calculation -- Prefers the cost xAI reports on the response over recomputing it locally - Uses the generic cost calculator which already handles tiered pricing correctly - Handles XAI-specific reasoning token billing (billed as part of completion tokens) +- Skips the web search surcharge when xAI reported a total that already covers it """ import math @@ -49,19 +49,6 @@ def _cost_reported_by_xai(usage: "Usage") -> float | None: return float(reported_cost) -def _split_reported_cost(model: str, usage: Usage, reported_cost: float) -> tuple[float, float]: - """xAI reports one total, so attribute it to input and output in the ratio our own token math gives.""" - try: - priced_input, priced_output = _token_math_cost_per_token(model, usage) - except Exception: # noqa: BLE001 # generic_cost_per_token raises bare Exception for unmapped models - return 0.0, reported_cost - priced_total: Final = priced_input + priced_output - if priced_total <= 0: - return 0.0, reported_cost - input_cost: Final = reported_cost * priced_input / priced_total - return input_cost, reported_cost - input_cost - - def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: """ Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens. @@ -74,9 +61,6 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - reported_cost: Final = _cost_reported_by_xai(usage) - if reported_cost is not None: - return _split_reported_cost(model, usage, reported_cost) return _token_math_cost_per_token(model, usage) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 53fd5c1cec2..5e18b8dec45 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -11878,7 +11878,89 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS } ) + +# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill, +# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed +# replies alike. A deployment the operator priced keeps the operator's rates on both. +_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected( + spend=0.037, input_cost=0.023125, output_cost=0.013875, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06) +_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected( + spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_CHUNK: Final = { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1, + "model": "gateway-reported-cost-fixture", +} + +GATEWAY_REPORTED_COST: Final = CostTrackingTestCase( + name="gateway-reported-cost", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False}, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1, + "model": "gateway-reported-cost-fixture", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": _GATEWAY_REPORTED_USAGE, + }, + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase( + name="gateway-reported-cost-stream", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "reported cost"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE}, + done=True, + ), + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( + GATEWAY_REPORTED_COST, + GATEWAY_REPORTED_COST_STREAM, + GATEWAY_REPORTED_COST_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12120,6 +12202,10 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( + StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index cd979528fe6..f620b88a86b 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_above_128k_tokens": 2.2e-07, "output_cost_per_token_above_128k_tokens": 8.8e-07, }, + "gateway-reported-cost-fixture": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + }, "openrouter/anthropic/claude-sonnet-5": { "litellm_provider": "openrouter", "mode": "chat", diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index c13cea51183..9e8cd46e5ea 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2401,14 +2401,14 @@ def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator(): ) == pytest.approx(0.00503) -def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): +def test_openai_compatible_streaming_cost_is_the_reported_cost(): import litellm from litellm.cost_calculator import ( get_response_cost_from_hidden_params, response_cost_calculator, ) - model = "openai/streams-cost-in-nanodollars" + model = "openai/streams-with-a-reported-cost" litellm.register_model( { model: { @@ -2423,7 +2423,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): id="chatcmpl-openai-compatible", model=model, choices=[], - usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000), + usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.037), ) CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai") @@ -2435,7 +2435,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): custom_llm_provider="openai", call_type="completion", optional_params={}, - ) == pytest.approx(2e-5) + ) == pytest.approx(0.037) def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch): diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 83c71479311..8812b227b8c 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -15,6 +15,7 @@ import pytest # Add the project root to Python path import litellm +from litellm.cost_calculator import cost_per_token as billed_cost_per_token from litellm.llms.perplexity.cost_calculator import ( cost_per_token as perplexity_cost_per_token, ) @@ -109,13 +110,13 @@ class TestPerplexityCostCalculator: "total_cost": 0.008, } - prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage + ) - # When Perplexity provides total_cost, we use it directly - # prompt_cost should be 0, completion_cost should be total_cost - assert prompt_cost == 0.0 - assert completion_cost == 0.008 - assert prompt_cost + completion_cost == 0.008 + # When Perplexity provides total_cost, that total is the bill + assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10) + assert prompt_cost > 0.0 and completion_cost > 0.0 def test_uses_perplexity_provided_cost_when_normalized_to_float(self): """ @@ -135,10 +136,11 @@ class TestPerplexityCostCalculator: usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) usage.cost = 0.008 - prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage + ) - assert prompt_cost == 0.0 - assert completion_cost == 0.008 + assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10) OFF_PEAK_MODEL = "sonar-off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" diff --git a/tests/unit/llms/xai/test_xai_cost_calculator.py b/tests/unit/llms/xai/test_xai_cost_calculator.py index 5219dba42ba..c1d2ee5f63d 100644 --- a/tests/unit/llms/xai/test_xai_cost_calculator.py +++ b/tests/unit/llms/xai/test_xai_cost_calculator.py @@ -6,6 +6,7 @@ import math import os import litellm +from litellm.cost_calculator import cost_per_token as billed_cost_per_token from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -115,7 +116,9 @@ class TestXAICostCalculator: model="grok-4.7", usage=Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) ) - prompt_cost, completion_cost = cost_per_token(model="grok-4.7", usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model="xai/grok-4.7", custom_llm_provider="xai", usage_object=usage + ) assert math.isclose(prompt_cost + completion_cost, 0.0037756, rel_tol=1e-10) assert math.isclose(prompt_cost / completion_cost, priced_prompt / priced_completion, rel_tol=1e-10) @@ -165,7 +168,10 @@ class TestXAICostCalculator: """A reported zero is a real answer, not a missing value.""" usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300, cost=0.0) - assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0) + assert billed_cost_per_token(model="xai/grok-4-latest", custom_llm_provider="xai", usage_object=usage) == ( + 0.0, + 0.0, + ) def test_custom_pricing_beats_the_reported_cost(self): response = ModelResponse( diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 27b25c1f98e..8702a79807e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10522,3 +10522,169 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry is not None assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) + + +_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037} +_REPORTED_COST_FIXTURE_MODEL: Final = "openai/reported-cost-fixture" +_REPORTED_COST_FIXTURE_RATES: Final = { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 3e-6, +} + + +def _reported_cost_logging(stream: bool, litellm_params: Mapping[str, object]) -> Logging: + logging_obj: Final = Logging( + model=_REPORTED_COST_FIXTURE_MODEL, + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="reported-cost", + function_id="reported-cost", + ) + logging_obj.update_from_kwargs( + kwargs={}, optional_params={}, litellm_params={"custom_llm_provider": "openai", **litellm_params} + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + return logging_obj + + +def _reported_cost_stream_chunk(content: str | None, finish_reason: str | None, usage: Usage | None) -> object: + return litellm.ModelResponseStream( + id="chatcmpl-reported", + model="reported-cost-fixture", + created=1, + object="chat.completion.chunk", + choices=[litellm.types.utils.StreamingChoices(index=0, delta=litellm.types.utils.Delta(content=content), finish_reason=finish_reason)], + usage=usage, + ) + + +def _bill_reported_cost(stream: bool, litellm_params: Mapping[str, object], hidden_params: Mapping[str, object]) -> float | None: + """The spend the proxy logs for one upstream reply carrying usage.cost, streamed or not.""" + logging_obj: Final = _reported_cost_logging(stream, litellm_params) + if stream: + response: Final[object] = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, Usage(**_REPORTED_USAGE)), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + else: + response = ModelResponse( + id="chatcmpl-reported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=Usage(**_REPORTED_USAGE), + ) + assert isinstance(response, ModelResponse) + response._hidden_params.update({"custom_llm_provider": "openai", **hidden_params}) + return logging_obj.response_cost_calculator(result=response) + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("mapped", [False, True]) +def test_provider_reported_usage_cost_is_billed_on_stream_and_non_stream( + monkeypatch: pytest.MonkeyPatch, stream: bool, mapped: bool +) -> None: + """Regression for #47533: a plain openai-compatible deployment billed usage.cost on streams and $0 on non-stream.""" + if mapped: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + else: + monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False) + + assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_deployment_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: + """Streams used to bill usage.cost even when the operator priced the deployment; both paths bill the operator's rates.""" + deployment_id: Final = "reported-cost-deployment-priced-by-the-operator" + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + ) + litellm_params: Final = { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 6e-6, + "model_info": {"id": deployment_id}, + } + + assert _bill_reported_cost(stream, litellm_params, {"model_id": deployment_id}) == pytest.approx( + 100 * 2e-6 + 20 * 6e-6 + ) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_base_model_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: + base_model: Final = "openai/reported-cost-base-model-fixture" + monkeypatch.setitem( + litellm.model_cost, + base_model, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 7e-6}, + ) + + assert _bill_reported_cost(stream, {"base_model": base_model}, {}) == pytest.approx(100 * 5e-6 + 20 * 7e-6) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_provider_reported_usage_cost_takes_discount_then_margin(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.2}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"percentage": 0.5, "fixed_amount": 0.001}}) + + assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037 * 0.8 * 1.5 + 0.001) + + +def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + usage: Final = Usage(**_REPORTED_USAGE) + + prompt_cost, completion_cost = cost_per_token( + model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=usage + ) + + priced_prompt: Final = 100 * 1e-6 + priced_completion: Final = 20 * 3e-6 + assert prompt_cost + completion_cost == pytest.approx(0.037) + assert prompt_cost == pytest.approx(0.037 * priced_prompt / (priced_prompt + priced_completion)) + + +def test_provider_reported_usage_cost_without_rates_is_all_output(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False) + + assert cost_per_token( + model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=Usage(**_REPORTED_USAGE) + ) == (0.0, 0.037) + + +@pytest.mark.parametrize("reported", [-0.037, float("nan"), float("inf"), True]) +def test_unusable_provider_reported_usage_cost_falls_back_to_the_token_math( + monkeypatch: pytest.MonkeyPatch, reported: object +) -> None: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + setattr(usage, "cost", reported) + + assert cost_per_token(model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=usage) == ( + pytest.approx(100 * 1e-6), + pytest.approx(20 * 3e-6), + ) + + +def test_caller_supplied_custom_cost_per_token_beats_the_provider_reported_usage_cost() -> None: + assert cost_per_token( + model=_REPORTED_COST_FIXTURE_MODEL, + custom_llm_provider="openai", + prompt_tokens=100, + completion_tokens=20, + usage_object=Usage(**_REPORTED_USAGE), + custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}, + ) == (pytest.approx(0.1), pytest.approx(0.04)) From b4ee6ef4f30ae53230a21651055c024c4fd322f3 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:20:53 +0000 Subject: [PATCH 02/26] fix(cost): stamp the billed cost over a reported usage.cost on streams The final streamed usage.cost a client sees is what the proxy billed, so an operator priced deployment shows its own rates instead of the upstream's number. Types region_name on cost_per_token. --- litellm/cost_calculator.py | 4 ++-- litellm/litellm_core_utils/streaming_chunk_builder_utils.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6ec8d00e37f..cacf130aeab 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -427,7 +427,7 @@ def cost_per_token( completion_tokens: int = 0, response_time_ms: float | None = 0.0, custom_llm_provider: str | None = None, - region_name=None, + region_name: str | None = None, ### CHARACTER PRICING ### prompt_characters: int | None = None, completion_characters: int | None = None, @@ -508,7 +508,7 @@ def _cost_per_token_from_rates( completion_tokens: int = 0, response_time_ms: float | None = 0.0, custom_llm_provider: str | None = None, - region_name=None, + region_name: str | None = None, ### CHARACTER PRICING ### prompt_characters: int | None = None, completion_characters: int | None = None, diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 4cdb9004619..3a5a2cbe914 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1413,8 +1413,6 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if isinstance(getattr(usage, "cost", None), (int, float)): - return computed_cost: Final = logging_obj.response_cost_calculator(result=response) if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: return From e0ee9620543639837ce0b048e896c7914b207d9b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:46:17 +0000 Subject: [PATCH 03/26] fix(cost): never re-bill a usage.cost litellm stamped on a stream as a provider report stamp_billed_usage_cost marks the response when the stream builder or the Responses stream stamps litellm's own bill into usage.cost, and completion_cost drops such a usage.cost before pricing, so a second pricing pass (MCP auto-execute rounds, uncached_price) does not add the margin twice. The xAI and Perplexity reported-cost tests go through the central calculator, with the token math falling back to an all-output split for an unmapped model. --- litellm/cost_calculator.py | 4 +-- litellm/litellm_core_utils/hidden_params.py | 14 +++++++++++ .../streaming_chunk_builder_utils.py | 4 +-- litellm/responses/streaming_iterator.py | 3 ++- .../test_perplexity_cost_calculator.py | 8 +++--- .../test_xai_responses_transformation.py | 6 ++--- .../llms/xai/test_xai_chat_transformation.py | 6 ++--- tests/unit/test_cost_calculator.py | 25 +++++++++++++++++++ 8 files changed, 56 insertions(+), 14 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index cacf130aeab..2eef729296c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -25,7 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping, provider_reported_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, usage_cost_stamped_by_litellm from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -1705,7 +1705,7 @@ def completion_cost( provider_usage_object: Final = get_usage_object(completion_response=completion_response) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) - if custom_pricing or base_model is not None + if custom_pricing or base_model is not None or usage_cost_stamped_by_litellm(completion_response) else provider_usage_object ) rerank_billed_units: RerankBilledUnits | None = None diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index a74337d2549..30082f1af7f 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -134,3 +134,17 @@ def get_or_create_hidden_params(obj: object) -> MutableMapping[str, object]: if hidden_params_mapping is not None: return hidden_params_mapping raise TypeError(f"unsupported hidden params storage: {type(hidden_params).__name__}") + + +USAGE_COST_STAMPED_BY_LITELLM: Final = "usage_cost_stamped_by_litellm" + + +def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None: + """usage.cost becomes what litellm billed, marked so the calculator never re-bills it as a provider's report.""" + setattr(usage, "cost", cost) + set_hidden_param(obj, USAGE_COST_STAMPED_BY_LITELLM, True) + + +def usage_cost_stamped_by_litellm(obj: object) -> bool: + hidden_params: Final = get_hidden_params(obj) + return hidden_params is not None and hidden_params.get(USAGE_COST_STAMPED_BY_LITELLM) is True diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 3a5a2cbe914..024ddf0bd91 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billed_usage_cost from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1416,7 +1416,7 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o computed_cost: Final = logging_obj.response_cost_calculator(result=response) if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: return - setattr(usage, "cost", computed_cost) + stamp_billed_usage_cost(response, usage, computed_cost) _set_hidden_response_cost(response, float(computed_cost)) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 50ab3e4f75a..0d840d136d0 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,6 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base +from litellm.litellm_core_utils.hidden_params import stamp_billed_usage_cost from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1757,7 +1758,7 @@ def stamp_responses_usage_cost( except Exception: return if isinstance(cost, (int, float)) and cost > 0: - setattr(usage_obj, "cost", cost) + stamp_billed_usage_cost(response_obj, usage_obj, cost) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 8812b227b8c..cd25b7a9c3d 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -228,7 +228,9 @@ class TestPerplexityCostCalculator: usage = Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200) usage.cost = {"total_cost": 0.00501} - prompt_cost, completion_cost = perplexity_cost_per_token(model=self.OFF_PEAK_MODEL, usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model=self.OFF_PEAK_MODEL, custom_llm_provider="perplexity", usage_object=usage + ) - assert prompt_cost == 0.0 - assert completion_cost == 0.00501 + assert math.isclose(prompt_cost + completion_cost, 0.00501, rel_tol=1e-10) + assert prompt_cost > 0.0 and completion_cost > 0.0 diff --git a/tests/unit/llms/xai/responses/test_xai_responses_transformation.py b/tests/unit/llms/xai/responses/test_xai_responses_transformation.py index 188ca453364..95ff98ac024 100644 --- a/tests/unit/llms/xai/responses/test_xai_responses_transformation.py +++ b/tests/unit/llms/xai/responses/test_xai_responses_transformation.py @@ -11,7 +11,7 @@ from unittest.mock import MagicMock, Mock import httpx -from litellm.llms.xai.cost_calculator import cost_per_token +from litellm.cost_calculator import cost_per_token from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( @@ -498,7 +498,7 @@ class TestXAIResponsesReportedCost: assert usage.cost == 0.0037756 chat_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage) - assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=chat_usage) == (0.0, 0.0037756) def test_streamed_reported_cost_reaches_the_cost_calculator(self): event = XAIResponsesAPIConfig().transform_streaming_response( @@ -520,7 +520,7 @@ class TestXAIResponsesReportedCost: assert isinstance(event, ResponseCompletedEvent) chat_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(event.response.usage) - assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=chat_usage) == (0.0, 0.0037756) def test_usage_without_a_reported_cost_is_left_alone(self): usage = self._transformed_usage({"input_tokens": 100, "output_tokens": 200, "total_tokens": 300}) diff --git a/tests/unit/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py index 69e620dbc8a..3aa35759662 100644 --- a/tests/unit/llms/xai/test_xai_chat_transformation.py +++ b/tests/unit/llms/xai/test_xai_chat_transformation.py @@ -13,7 +13,7 @@ from litellm.llms.xai.chat.transformation import ( XAIChatCompletionStreamingHandler, XAIChatConfig, ) -from litellm.llms.xai.cost_calculator import cost_per_token +from litellm.cost_calculator import cost_per_token from litellm.types.utils import ( CompletionTokensDetailsWrapper, ModelResponse, @@ -433,7 +433,7 @@ class TestXAIReportedCost: ) assert usage.cost == 0.0037756 - assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=usage) == (0.0, 0.0037756) def test_usage_without_a_reported_cost_is_left_alone(self): usage = self._transformed_usage({"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}) @@ -481,7 +481,7 @@ class TestXAIReportedCost: assembled = litellm.stream_chunk_builder(chunks=[parsed]) assert assembled.usage.cost == 0.0037756 - assert cost_per_token(model="grok-4-latest", usage=assembled.usage) == ( + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=assembled.usage) == ( 0.0, 0.0037756, ) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 8702a79807e..81b00f3225d 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10641,6 +10641,31 @@ def test_provider_reported_usage_cost_takes_discount_then_margin(monkeypatch: py assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037 * 0.8 * 1.5 + 0.001) +def test_a_usage_cost_litellm_stamped_on_a_stream_is_not_billed_again_as_a_provider_report( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.002}}) + logging_obj: Final = _reported_cost_logging(True, {}) + token_math_plus_margin: Final = 100 * 1e-6 + 20 * 3e-6 + 0.002 + + response: Final[object] = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + + assert isinstance(response, ModelResponse) + assert response.usage is not None and response.usage.cost == pytest.approx(token_math_plus_margin) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx( + token_math_plus_margin + ) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: From 2c0e47ba7e177244fee032ec60129b61095752b9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:07:22 +0000 Subject: [PATCH 04/26] fix(cost): keep the provider's usage.cost behind a stamped bill and read it on every pricing pass stamp_billed_usage_cost records the provider's own usage.cost in hidden params before writing litellm's bill over it, and completion_cost prices from that recorded figure, so a second pass (uncached_price, MCP rounds) bills the same number as the first instead of re-billing the stamped total or losing the report on an unmapped model. cost_per_token reads the reported cost per provider, so OpenCode Go's balance-hit usage.cost of 0 no longer zeroes a row. --- litellm/cost_calculator.py | 24 +++++++++++++++------ litellm/litellm_core_utils/hidden_params.py | 21 +++++++++++------- tests/unit/test_main.py | 7 +++--- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 2eef729296c..d37bd7cc67e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -25,7 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping, provider_reported_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, usage_cost_stamped_by_litellm +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, provider_stated_usage_cost from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -408,8 +408,8 @@ def _per_second_pricing_cost( return resolved_cost_per_second * seconds, 0.0 -def _billable_reported_cost(usage_object: Usage | None) -> float | None: - return _billable_header_cost(provider_reported_usage_cost(usage_object, None)) +def _billable_reported_cost(usage_object: Usage | None, custom_llm_provider: str | None) -> float | None: + return _billable_header_cost(provider_reported_usage_cost(usage_object, custom_llm_provider)) def _split_reported_cost(reported_cost: float, priced: tuple[float, float] | None) -> tuple[float, float]: @@ -465,7 +465,7 @@ def cost_per_token( Without a reported cost the token math is the bill. """ priced_by_caller: Final = custom_cost_per_token is not None or custom_cost_per_second is not None - reported_cost: Final = None if priced_by_caller else _billable_reported_cost(usage_object) + reported_cost: Final = None if priced_by_caller else _billable_reported_cost(usage_object, custom_llm_provider) rated: Final = partial( _cost_per_token_from_rates, model=model, @@ -1560,6 +1560,16 @@ def _store_cost_breakdown_in_logging_obj( # Don't fail the main cost calculation if breakdown storage fails +def _with_provider_stated_cost(completion_response: object, usage: Usage | None) -> Usage | None: + if usage is None: + return usage + stamped_cost: Final = getattr(usage, "cost", None) + stated_cost: Final = provider_stated_usage_cost(completion_response, stamped_cost) + if stated_cost is stamped_cost: + return usage + return usage.model_copy(update=MappingProxyType({"cost": stated_cost})) + + def _without_provider_stated_cost(usage: Usage | None) -> Usage | None: if usage is None or getattr(usage, "cost", None) is None: return usage @@ -1702,10 +1712,12 @@ def completion_cost( cache_creation_input_tokens: int | None = None cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 - provider_usage_object: Final = get_usage_object(completion_response=completion_response) + provider_usage_object: Final = _with_provider_stated_cost( + completion_response, get_usage_object(completion_response=completion_response) + ) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) - if custom_pricing or base_model is not None or usage_cost_stamped_by_litellm(completion_response) + if custom_pricing or base_model is not None else provider_usage_object ) rerank_billed_units: RerankBilledUnits | None = None diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index 30082f1af7f..ed493394f03 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -136,15 +136,20 @@ def get_or_create_hidden_params(obj: object) -> MutableMapping[str, object]: raise TypeError(f"unsupported hidden params storage: {type(hidden_params).__name__}") -USAGE_COST_STAMPED_BY_LITELLM: Final = "usage_cost_stamped_by_litellm" +USAGE_COST_BEFORE_LITELLM_STAMP: Final = "usage_cost_before_litellm_stamp" def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None: - """usage.cost becomes what litellm billed, marked so the calculator never re-bills it as a provider's report.""" - setattr(usage, "cost", cost) - set_hidden_param(obj, USAGE_COST_STAMPED_BY_LITELLM, True) - - -def usage_cost_stamped_by_litellm(obj: object) -> bool: + """usage.cost becomes what litellm billed; the provider's own figure is kept so a later pricing pass bills the same.""" hidden_params: Final = get_hidden_params(obj) - return hidden_params is not None and hidden_params.get(USAGE_COST_STAMPED_BY_LITELLM) is True + if hidden_params is None or USAGE_COST_BEFORE_LITELLM_STAMP not in hidden_params: + set_hidden_param(obj, USAGE_COST_BEFORE_LITELLM_STAMP, getattr(usage, "cost", None)) + setattr(usage, "cost", cost) + + +def provider_stated_usage_cost(obj: object, usage_cost: object) -> object: + """What the provider itself put in usage.cost, looking past a bill litellm stamped there.""" + hidden_params: Final = get_hidden_params(obj) + if hidden_params is None or USAGE_COST_BEFORE_LITELLM_STAMP not in hidden_params: + return usage_cost + return hidden_params.get(USAGE_COST_BEFORE_LITELLM_STAMP) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 70b268bf2a2..1faf6dc12ad 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5062,7 +5062,7 @@ def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): assert response._hidden_params["response_cost"] == pytest.approx(0.000704) -def test_stream_chunk_builder_leaves_xai_reported_cost_to_the_calculator(monkeypatch: pytest.MonkeyPatch): +def test_stream_chunk_builder_stamps_the_margin_priced_bill_over_the_xai_reported_cost(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5}) usage_chunk: Final = _stream_builder_text_chunk("grok-4", "") usage_chunk.usage = Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7, cost=0.42) @@ -5078,9 +5078,10 @@ def test_stream_chunk_builder_leaves_xai_reported_cost_to_the_calculator(monkeyp ) assert response is not None - assert getattr(response.usage, "cost", None) == pytest.approx(0.42) - assert response._hidden_params.get("response_cost") is None + assert getattr(response.usage, "cost", None) == pytest.approx(0.63) + assert response._hidden_params.get("response_cost") == pytest.approx(0.63) assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.63) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(0.63) def test_speech_mistral_dispatches_and_decodes_audio(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): From 1a890ead5edac2a7a549502501c8fef75c3dbd70 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:26:05 +0000 Subject: [PATCH 05/26] fix(cost): stamp a zero bill over a reported usage.cost on streams A 100 percent discount or zero operator rates bill 0, and the final streamed usage.cost now says so instead of leaving the upstream's number on the chunk. --- .../streaming_chunk_builder_utils.py | 2 +- tests/unit/test_cost_calculator.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 024ddf0bd91..443c39b647e 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1414,7 +1414,7 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o if logging_obj is None: return computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: + if not isinstance(computed_cost, (int, float)) or computed_cost < 0: return stamp_billed_usage_cost(response, usage, computed_cost) _set_hidden_response_cost(response, float(computed_cost)) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 81b00f3225d..a52efc6b41e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10666,6 +10666,25 @@ def test_a_usage_cost_litellm_stamped_on_a_stream_is_not_billed_again_as_a_provi ) +def test_a_zero_bill_is_stamped_over_the_provider_reported_usage_cost_on_a_stream(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 1.0}) + logging_obj: Final = _reported_cost_logging(True, {}) + + response: Final[object] = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, Usage(**_REPORTED_USAGE)), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + + assert isinstance(response, ModelResponse) + assert response.usage is not None and response.usage.cost == 0.0 + assert logging_obj.response_cost_calculator(result=response) == 0.0 + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: From 0fd6d5510dac0fd76c6b063d97dd84f0172c8859 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:50:06 +0000 Subject: [PATCH 06/26] fix(cost): stamp a zero bill only over a usage.cost the provider reported An unpriceable stream keeps no usage.cost, as before. --- litellm/litellm_core_utils/streaming_chunk_builder_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 443c39b647e..4c983d6b2c5 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1416,6 +1416,8 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o computed_cost: Final = logging_obj.response_cost_calculator(result=response) if not isinstance(computed_cost, (int, float)) or computed_cost < 0: return + if computed_cost == 0 and getattr(usage, "cost", None) is None: + return stamp_billed_usage_cost(response, usage, computed_cost) _set_hidden_response_cost(response, float(computed_cost)) From 3490128d7d37db627782c8da0ced97e5002d3ad2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:53:25 +0000 Subject: [PATCH 07/26] fix(cost): carry the provider's usage.cost onto the Responses stream logging copy The logging copy of a streamed Responses reply is rebuilt from headers only, which dropped the figure the stamp keeps, so the stamped bill was priced again as a provider report. --- litellm/responses/streaming_iterator.py | 11 ++++-- .../unit/responses/test_streaming_iterator.py | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0d840d136d0..3ef9c7bee4d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,7 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base -from litellm.litellm_core_utils.hidden_params import stamp_billed_usage_cost +from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billed_usage_cost from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -676,10 +676,17 @@ class BaseResponsesAPIStreamingIterator: headers: Final[Mapping[str, object]] = processed if isinstance(processed, Mapping) else EMPTY_MAPPING raw_headers: Final[Mapping[str, object]] = raw if isinstance(raw, Mapping) else EMPTY_MAPPING # rebuild by value and let existing keys win: sharing the source dicts would alias what the proxy - # splats into the client's HTTP headers, and copying non-header keys would carry response_cost + # splats into the client's HTTP headers, and copying non-header keys would carry response_cost. + # The provider's own usage.cost travels along, or the stamped bill would be priced as a provider report + stamp: Final[Mapping[str, object]] = ( + {USAGE_COST_BEFORE_LITELLM_STAMP: source[USAGE_COST_BEFORE_LITELLM_STAMP]} + if USAGE_COST_BEFORE_LITELLM_STAMP in source + else EMPTY_MAPPING + ) target.hidden_params = { "additional_headers": {**headers}, "headers": {**raw_headers}, + **stamp, **existing, } diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 6c3e6177503..5716f46c210 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -2781,3 +2781,39 @@ async def test_websocket_turns_logging_survives_garbage_collection(terminal_even await _run_websocket_turn(logger, terminal_event) assert await _logging_tasks_left_after_garbage_collection(logger) == 1 + + +def test_logging_copy_keeps_the_provider_usage_cost_behind_a_stamped_bill() -> None: + from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP + from litellm.responses.streaming_iterator import _logging_copy + + logging_obj = LiteLLMLoggingObj( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="aresponses", + start_time=datetime.now(), + litellm_call_id="stamped-copy", + function_id="stamped-copy", + ) + iterator = _make_iterator(sse_events=[], logging_obj=logging_obj) + billed = ResponsesAPIResponse( + id="resp_stamped", + created_at=1, + status="completed", + model="gpt-4o-mini", + object="response", + output=[], + usage={"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.63}, + ) + billed.hidden_params = {USAGE_COST_BEFORE_LITELLM_STAMP: 0.42, "response_cost": 0.63} + iterator.completed_response = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=billed + ) + logging_response = _logging_copy(iterator.completed_response) + + iterator._restore_provider_response_headers(logging_response) + + copied = logging_response.response.hidden_params + assert copied[USAGE_COST_BEFORE_LITELLM_STAMP] == 0.42 + assert "response_cost" not in copied From a20609a6f2419ad8577c3cd09b207798fc63bedc Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:37:04 +0000 Subject: [PATCH 08/26] fix(cost): show litellm's bill in usage.cost on chat non-stream and both Responses paths A priced deployment showed its bill only on chat streams. The metadata pass now stamps the bill onto ModelResponse and ResponsesAPIResponse usage, the Responses stream stamp overwrites the upstream's number too, all through one stamp_billable_usage_cost guard, and the integration suite asserts a JSON reply's usage.cost equals the spend row and gains /v1/responses twins of the reported-cost cases --- litellm/litellm_core_utils/hidden_params.py | 10 ++ .../llm_response_utils/response_metadata.py | 26 +++-- .../streaming_chunk_builder_utils.py | 11 +- litellm/responses/streaming_iterator.py | 7 +- .../cost_calculation/assertions.py | 16 +++ .../cost_calculation/cases/openai.py | 80 ++++++++++++++ .../unit/responses/test_streaming_iterator.py | 27 ++++- tests/unit/test_cost_calculator.py | 102 ++++++++++++++++++ 8 files changed, 257 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index ed493394f03..7281b138894 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -147,6 +147,16 @@ def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None: setattr(usage, "cost", cost) +def stamp_billable_usage_cost(obj: object, usage: object, cost: object) -> float | None: + """Stamp a priced bill, a $0 one included once the provider put a number there; an unpriceable reply keeps its usage.""" + if isinstance(cost, bool) or not isinstance(cost, (int, float)) or cost < 0: + return None + if cost == 0 and getattr(usage, "cost", None) is None: + return None + stamp_billed_usage_cost(obj, usage, float(cost)) + return float(cost) + + def provider_stated_usage_cost(obj: object, usage_cost: object) -> object: """What the provider itself put in usage.cost, looking past a bill litellm stamped there.""" hidden_params: Final = get_hidden_params(obj) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ace57a64aae..4379b937394 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -8,13 +8,16 @@ from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers +from litellm.litellm_core_utils.hidden_params import stamp_billable_usage_cost from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( EmbeddingResponse, HiddenParams, ModelResponse, TranscriptionResponse, + Usage, ) @@ -159,16 +162,18 @@ class ResponseMetadata: ## ADD OTHER HIDDEN PARAMS model_info: Final = kwargs.get("model_info", {}) or {} model_id: Final = model_info.get("id", None) + response_cost: Final = logging_obj.response_cost_calculator( + result=self.result, + litellm_model_name=model, + router_model_id=model_id, + cache_hit=cache_hit, + ) + self._stamp_billed_usage_cost(response_cost) new_params: Final = { "litellm_call_id": getattr(logging_obj, "litellm_call_id", None), "api_base": get_api_base(model=model or "", optional_params=kwargs), "model_id": model_id, - "response_cost": logging_obj.response_cost_calculator( - result=self.result, - litellm_model_name=model, - router_model_id=model_id, - cache_hit=cache_hit, - ), + "response_cost": response_cost, "additional_headers": process_response_headers( self._get_additional_headers_from_hidden_params() or {}, preserve_litellm_internal_headers=True, @@ -179,6 +184,15 @@ class ResponseMetadata: } self._update_hidden_params(new_params) + def _stamp_billed_usage_cost(self, response_cost: object) -> None: + """A client reads what litellm billed from usage.cost, streamed or not, priced deployment or not.""" + if not isinstance(self.result, (ModelResponse, ResponsesAPIResponse)): + return + usage: Final = getattr(self.result, "usage", None) + if not isinstance(usage, (Usage, ResponseAPIUsage)): + return + stamp_billable_usage_cost(self.result, usage, response_cost) + def _update_hidden_params(self, new_params: Mapping[str, object]) -> None: """ Update hidden params - handles when self._hidden_params is a dict or HiddenParams object diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 4c983d6b2c5..3425871a408 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billed_usage_cost +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billable_usage_cost from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1413,13 +1413,10 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if not isinstance(computed_cost, (int, float)) or computed_cost < 0: + billed: Final = stamp_billable_usage_cost(response, usage, logging_obj.response_cost_calculator(result=response)) + if billed is None: return - if computed_cost == 0 and getattr(usage, "cost", None) is None: - return - stamp_billed_usage_cost(response, usage, computed_cost) - _set_hidden_response_cost(response, float(computed_cost)) + _set_hidden_response_cost(response, billed) def _field_of(value: object, name: str) -> object: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 3ef9c7bee4d..ca6bbf0046c 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,7 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base -from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billed_usage_cost +from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billable_usage_cost from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1758,14 +1758,11 @@ def stamp_responses_usage_cost( if usage_obj is None: return response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives - if isinstance(getattr(usage_obj, "cost", None), (int, float)): - return try: cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj) except Exception: return - if isinstance(cost, (int, float)) and cost > 0: - stamp_billed_usage_cost(response_obj, usage_obj, cost) + stamp_billable_usage_cost(response_obj, usage_obj, cost) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/integration/cost_calculation/assertions.py b/tests/integration/cost_calculation/assertions.py index 6e2af74d1a2..f37bb32e3b3 100644 --- a/tests/integration/cost_calculation/assertions.py +++ b/tests/integration/cost_calculation/assertions.py @@ -55,6 +55,18 @@ def assert_streamed_usage_cost(case_name: str, endpoint: str, response_text: str assert approx_equal(seen, spend), f"{case_name}: streamed usage.cost {seen} != spend {spend}" +def assert_json_usage_cost(case_name: str, endpoint: str, response: httpx.Response, spend: float) -> None: + """A JSON reply's usage.cost is what the row billed, the upstream's own figure included, so a client and the + spend log never disagree on a priced deployment.""" + if endpoint not in STREAMED_COST_ENDPOINTS: + return + usage: Final = JSON_OBJECT.validate_json(response.text).get("usage") + cost: Final = usage.get("cost") if isinstance(usage, dict) else None + if not isinstance(cost, (int, float)) or isinstance(cost, bool): + return + assert approx_equal(float(cost), spend), f"{case_name}: usage.cost {cost} != spend {spend}" + + def assert_breakdown( case_name: str, response_content_type: str, @@ -160,6 +172,8 @@ def assert_exact( assert_total_is_sum_of_components(row, breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, expected.spend) + if response is not None and response_content_type == "application/json": + assert_json_usage_cost(case_name, response.request.url.path, response, expected.spend) def assert_recount( @@ -201,3 +215,5 @@ def assert_recount( assert_total_is_sum_of_components(row, row.breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, row.spend) + if response is not None and response_content_type == "application/json": + assert_json_usage_cost(case_name, response.request.url.path, response, row.spend) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 5e18b8dec45..fb7a90a634a 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -2,6 +2,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +from collections.abc import Mapping from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( @@ -11956,11 +11957,85 @@ GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STRE } ) +_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = { + "id": "msg_gateway", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], +} +_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gateway-reported-cost-fixture", + "output": [_GATEWAY_RESPONSES_ITEM], + "usage": _GATEWAY_RESPONSES_USAGE, +} + +GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase( + name="gateway-reported-cost-responses", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/responses", + request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}}, + response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-stream", + "request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True}, + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "response.created", + "sequence_number": 0, + "response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_gateway", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY}, + ), + ), + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GATEWAY_REPORTED_COST, GATEWAY_REPORTED_COST_STREAM, GATEWAY_REPORTED_COST_OPERATOR_PRICED, GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_RESPONSES, + GATEWAY_REPORTED_COST_RESPONSES_STREAM, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12206,6 +12281,11 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM ), + StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 5716f46c210..8d624a186ad 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -851,17 +851,36 @@ def test_stamp_responses_usage_cost_stamps_computed_cost(): logging_obj.response_cost_calculator.assert_called_once_with(result=response) -def test_stamp_responses_usage_cost_keeps_provider_reported_cost(): +def test_stamp_responses_usage_cost_puts_the_bill_over_the_provider_reported_cost(): + from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, provider_stated_usage_cost from litellm.responses.streaming_iterator import stamp_responses_usage_cost response = _responses_api_response_with_usage() setattr(response.usage, "cost", 0.5) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 0.00032 stamp_responses_usage_cost(response, logging_obj) - assert getattr(response.usage, "cost", None) == pytest.approx(0.5) - logging_obj.response_cost_calculator.assert_not_called() + assert getattr(response.usage, "cost", None) == pytest.approx(0.00032) + assert provider_stated_usage_cost(response, response.usage.cost) == pytest.approx(0.5) + assert response._hidden_params[USAGE_COST_BEFORE_LITELLM_STAMP] == pytest.approx(0.5) + + +def test_stamp_responses_usage_cost_stamps_a_zero_bill_only_over_a_provider_reported_cost(): + from litellm.responses.streaming_iterator import stamp_responses_usage_cost + + unpriced = _responses_api_response_with_usage() + reported = _responses_api_response_with_usage() + setattr(reported.usage, "cost", 0.5) + logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 0.0 + + stamp_responses_usage_cost(unpriced, logging_obj) + stamp_responses_usage_cost(reported, logging_obj) + + assert getattr(unpriced.usage, "cost", None) is None + assert getattr(reported.usage, "cost", None) == 0.0 def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: @@ -889,13 +908,13 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): } ) logging_obj = Mock(spec=LiteLLMLoggingObj) + logging_obj.response_cost_calculator.return_value = 3e-05 stamp_responses_usage_cost(response, logging_obj) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) assert response.usage.output_tokens_details.reasoning_tokens == 117 - logging_obj.response_cost_calculator.assert_not_called() def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index a52efc6b41e..83724d7a2fc 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -24,6 +24,8 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.llm_response_utils.response_metadata import update_response_metadata +from litellm.responses.streaming_iterator import stamp_responses_usage_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, ) @@ -10685,6 +10687,106 @@ def test_a_zero_bill_is_stamped_over_the_provider_reported_usage_cost_on_a_strea assert logging_obj.response_cost_calculator(result=response) == 0.0 +_PRICED_DEPLOYMENT_ID: Final = "reported-cost-deployment-priced-by-the-operator" +_PRICED_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType( + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 6e-6, + "metadata": {"model_info": {"id": _PRICED_DEPLOYMENT_ID}}, + } +) + + +def _reported_cost_usage() -> Usage: + return Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037) + + +def _priced_logging(stream: bool, call_type: str) -> Logging: + """The logging object the proxy hands a priced deployment's call, the deployment under metadata.model_info.""" + logging_obj: Final = _reported_cost_logging(stream, _PRICED_LITELLM_PARAMS) + logging_obj.call_type = call_type + return logging_obj + + +def _reported_cost_responses_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_reported", + created_at=1, + status="completed", + model="reported-cost-fixture", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=0.037), + ) + + +def _client_visible_reply(route: str) -> tuple[object, Logging]: + """The object a client of the given route reads usage.cost from, after litellm priced it.""" + match route: + case "chat": + logging_obj: Logging = _priced_logging(False, "completion") + response: object = ModelResponse( + id="chatcmpl-reported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=_reported_cost_usage(), + ) + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + case "chat-stream": + logging_obj = _priced_logging(True, "completion") + response = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, _reported_cost_usage()), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + case "responses": + logging_obj = _priced_logging(False, "aresponses") + response = _reported_cost_responses_response() + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + case _: + logging_obj = _priced_logging(True, "aresponses") + response = _reported_cost_responses_response() + stamp_responses_usage_cost(response, logging_obj) + return response, logging_obj + + +@pytest.mark.parametrize("route", ["chat", "chat-stream", "responses", "responses-stream"]) +def test_the_client_reads_what_litellm_billed_from_usage_cost_on_every_route( + monkeypatch: pytest.MonkeyPatch, route: str +) -> None: + """A priced deployment showed its bill on chat streams only; the other three routes still showed the upstream's number.""" + monkeypatch.setitem( + litellm.model_cost, + _PRICED_DEPLOYMENT_ID, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + ) + bill: Final = 100 * 2e-6 + 20 * 6e-6 + + response, logging_obj = _client_visible_reply(route) + + usage: Final = getattr(response, "usage", None) + assert usage is not None and usage.cost == pytest.approx(bill) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: From e4b9584ea65ce6b0cd4d157553cd799d54ba0904 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:46:44 +0000 Subject: [PATCH 09/26] fix(cost): stamp the bill on a plain reply only where the upstream reported usage.cost A reply whose upstream sent no usage.cost keeps its usage untouched, as on main, so SDK callers comparing the usage dict to the upstream's see no added key and logging payloads gain no additional_usage_values.cost --- .../llm_response_utils/response_metadata.py | 5 ++-- tests/unit/test_cost_calculator.py | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 4379b937394..9a548018bb4 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -185,11 +185,12 @@ class ResponseMetadata: self._update_hidden_params(new_params) def _stamp_billed_usage_cost(self, response_cost: object) -> None: - """A client reads what litellm billed from usage.cost, streamed or not, priced deployment or not.""" + """A usage.cost the upstream reported becomes what litellm billed, so a client and the spend log agree on a + priced deployment; a reply without one keeps its usage as the upstream sent it.""" if not isinstance(self.result, (ModelResponse, ResponsesAPIResponse)): return usage: Final = getattr(self.result, "usage", None) - if not isinstance(usage, (Usage, ResponseAPIUsage)): + if not isinstance(usage, (Usage, ResponseAPIUsage)) or getattr(usage, "cost", None) is None: return stamp_billable_usage_cost(self.result, usage, response_cost) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 83724d7a2fc..c4266ec3e75 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10787,6 +10787,33 @@ def test_the_client_reads_what_litellm_billed_from_usage_cost_on_every_route( assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) +def test_a_plain_reply_without_an_upstream_usage_cost_keeps_none_after_pricing(monkeypatch: pytest.MonkeyPatch) -> None: + """Clients comparing the usage dict to the upstream's byte for byte must not find a cost litellm added.""" + monkeypatch.setitem( + litellm.model_cost, + _PRICED_DEPLOYMENT_ID, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + ) + response: Final = ModelResponse( + id="chatcmpl-unreported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120), + ) + + update_response_metadata( + result=response, + logging_obj=_priced_logging(False, "completion"), + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + + assert response.usage is not None and getattr(response.usage, "cost", None) is None + assert response._hidden_params["response_cost"] == pytest.approx(100 * 2e-6 + 20 * 6e-6) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: From df818a8e620147b813a9925334c99d7ee19cad3c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:50:21 +0000 Subject: [PATCH 10/26] feat(cost): include_cost_in_streaming_usage puts the bill on plain replies without an upstream usage.cost too The switch that injects litellm's bill into streamed usage now gives a plain chat or Responses reply the same usage.cost; off by default, so a reply without an upstream cost keeps its usage as sent --- .../llm_response_utils/response_metadata.py | 10 ++++-- tests/unit/test_cost_calculator.py | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 9a548018bb4..2c74cb1c0dd 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -4,6 +4,8 @@ from functools import reduce from typing import Final import httpx + +import litellm from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING @@ -186,11 +188,15 @@ class ResponseMetadata: def _stamp_billed_usage_cost(self, response_cost: object) -> None: """A usage.cost the upstream reported becomes what litellm billed, so a client and the spend log agree on a - priced deployment; a reply without one keeps its usage as the upstream sent it.""" + priced deployment. A reply without one keeps its usage as the upstream sent it unless the operator asked for + the bill in every usage block with ``litellm.include_cost_in_streaming_usage``, the same switch that puts it + into streamed usage.""" if not isinstance(self.result, (ModelResponse, ResponsesAPIResponse)): return usage: Final = getattr(self.result, "usage", None) - if not isinstance(usage, (Usage, ResponseAPIUsage)) or getattr(usage, "cost", None) is None: + if not isinstance(usage, (Usage, ResponseAPIUsage)): + return + if getattr(usage, "cost", None) is None and not litellm.include_cost_in_streaming_usage: return stamp_billable_usage_cost(self.result, usage, response_cost) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index c4266ec3e75..ad4e0749ce9 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10814,6 +10814,38 @@ def test_a_plain_reply_without_an_upstream_usage_cost_keeps_none_after_pricing(m assert response._hidden_params["response_cost"] == pytest.approx(100 * 2e-6 + 20 * 6e-6) +def test_include_cost_in_streaming_usage_puts_the_bill_on_a_plain_reply_without_an_upstream_usage_cost( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The switch that injects the bill into streamed usage gives plain replies the same key.""" + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) + monkeypatch.setitem( + litellm.model_cost, + _PRICED_DEPLOYMENT_ID, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + ) + response: Final = ModelResponse( + id="chatcmpl-unreported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120), + ) + logging_obj: Final = _priced_logging(False, "completion") + + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + + bill: Final = 100 * 2e-6 + 20 * 6e-6 + assert response.usage is not None and response.usage.cost == pytest.approx(bill) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: From 15670c78e853cbb4b2bab30cf27e1870a494a9fa Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:53:36 +0000 Subject: [PATCH 11/26] chore(cost): sort the litellm import in response_metadata --- .../litellm_core_utils/llm_response_utils/response_metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 2c74cb1c0dd..216f2b6a45c 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -4,10 +4,9 @@ from functools import reduce from typing import Final import httpx - -import litellm from pydantic import TypeAdapter, ValidationError +import litellm from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers from litellm.litellm_core_utils.hidden_params import stamp_billable_usage_cost From de17bca0ac9c49261ffe9a2a4546bd5c48369b94 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:19:19 +0000 Subject: [PATCH 12/26] fix(cost): keep the provider's usage.cost marker through the metadata pass ResponseMetadata builds a fresh hidden-params dict for a reply that arrived with none, so the marker the stamp wrote was dropped by apply() and a second pricing pass applied the margin twice --- .../llm_response_utils/response_metadata.py | 13 +++++++-- tests/unit/test_cost_calculator.py | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 216f2b6a45c..ef47e2f0d9f 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -9,7 +9,11 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers -from litellm.litellm_core_utils.hidden_params import stamp_billable_usage_cost +from litellm.litellm_core_utils.hidden_params import ( + USAGE_COST_BEFORE_LITELLM_STAMP, + get_hidden_params, + stamp_billable_usage_cost, +) from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse @@ -197,7 +201,12 @@ class ResponseMetadata: return if getattr(usage, "cost", None) is None and not litellm.include_cost_in_streaming_usage: return - stamp_billable_usage_cost(self.result, usage, response_cost) + if stamp_billable_usage_cost(self.result, usage, response_cost) is None: + return + stamped: Final = get_hidden_params(self.result) + if stamped is None or USAGE_COST_BEFORE_LITELLM_STAMP not in stamped: + return + self._update_hidden_params({USAGE_COST_BEFORE_LITELLM_STAMP: stamped[USAGE_COST_BEFORE_LITELLM_STAMP]}) def _update_hidden_params(self, new_params: Mapping[str, object]) -> None: """ diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index ad4e0749ce9..b6ff6cadb8d 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10846,6 +10846,34 @@ def test_include_cost_in_streaming_usage_puts_the_bill_on_a_plain_reply_without_ assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) +def test_a_bill_stamped_on_a_plain_reply_survives_the_metadata_pass_for_a_second_pricing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The metadata pass installs a fresh hidden-params dict on a reply that arrived with none, which dropped the + provider's figure the stamp keeps and let a second pricing pass apply the margin twice.""" + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"percentage": 0.5}}) + monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False) + logging_obj: Final = _reported_cost_logging(False, {}) + response: Final = ModelResponse( + id="chatcmpl-reported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=_reported_cost_usage(), + ) + + update_response_metadata( + result=response, + logging_obj=logging_obj, + model=_REPORTED_COST_FIXTURE_MODEL, + kwargs={}, + start_time=datetime.datetime(2026, 1, 1), + end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), + ) + + assert response.usage is not None and response.usage.cost == pytest.approx(0.037 * 1.5) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(0.037 * 1.5) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: From 332ba6f9fbd0faff4c9da65e3b61b013a2de50ce Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:37:28 +0000 Subject: [PATCH 13/26] refactor(cost): write the bill into usage.cost once at the proxy egress instead of inside the pipeline --- litellm/cost_calculator.py | 16 +- litellm/litellm_core_utils/hidden_params.py | 29 - .../llm_response_utils/response_metadata.py | 41 +- .../streaming_chunk_builder_utils.py | 14 +- litellm/proxy/common_request_processing.py | 11 +- litellm/proxy/proxy_server.py | 20 +- litellm/proxy/usage_cost_egress.py | 77 +++ .../streaming_iterator.py | 4 +- litellm/responses/streaming_iterator.py | 30 +- tests/unit/proxy/test_usage_cost_egress.py | 200 +++++++ .../unit/responses/test_streaming_iterator.py | 182 ++---- tests/unit/test_cost_calculator.py | 519 +++++------------- 12 files changed, 530 insertions(+), 613 deletions(-) create mode 100644 litellm/proxy/usage_cost_egress.py create mode 100644 tests/unit/proxy/test_usage_cost_egress.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 382943db244..cf28dbcd17c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -25,7 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping, provider_reported_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, provider_stated_usage_cost +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -1560,16 +1560,6 @@ def _store_cost_breakdown_in_logging_obj( # Don't fail the main cost calculation if breakdown storage fails -def _with_provider_stated_cost(completion_response: object, usage: Usage | None) -> Usage | None: - if usage is None: - return usage - stamped_cost: Final = getattr(usage, "cost", None) - stated_cost: Final = provider_stated_usage_cost(completion_response, stamped_cost) - if stated_cost is stamped_cost: - return usage - return usage.model_copy(update=MappingProxyType({"cost": stated_cost})) - - def _without_provider_stated_cost(usage: Usage | None) -> Usage | None: if usage is None or getattr(usage, "cost", None) is None: return usage @@ -1712,9 +1702,7 @@ def completion_cost( cache_creation_input_tokens: int | None = None cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 - provider_usage_object: Final = _with_provider_stated_cost( - completion_response, get_usage_object(completion_response=completion_response) - ) + provider_usage_object: Final = get_usage_object(completion_response=completion_response) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) if custom_pricing or base_model is not None diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index 7281b138894..a74337d2549 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -134,32 +134,3 @@ def get_or_create_hidden_params(obj: object) -> MutableMapping[str, object]: if hidden_params_mapping is not None: return hidden_params_mapping raise TypeError(f"unsupported hidden params storage: {type(hidden_params).__name__}") - - -USAGE_COST_BEFORE_LITELLM_STAMP: Final = "usage_cost_before_litellm_stamp" - - -def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None: - """usage.cost becomes what litellm billed; the provider's own figure is kept so a later pricing pass bills the same.""" - hidden_params: Final = get_hidden_params(obj) - if hidden_params is None or USAGE_COST_BEFORE_LITELLM_STAMP not in hidden_params: - set_hidden_param(obj, USAGE_COST_BEFORE_LITELLM_STAMP, getattr(usage, "cost", None)) - setattr(usage, "cost", cost) - - -def stamp_billable_usage_cost(obj: object, usage: object, cost: object) -> float | None: - """Stamp a priced bill, a $0 one included once the provider put a number there; an unpriceable reply keeps its usage.""" - if isinstance(cost, bool) or not isinstance(cost, (int, float)) or cost < 0: - return None - if cost == 0 and getattr(usage, "cost", None) is None: - return None - stamp_billed_usage_cost(obj, usage, float(cost)) - return float(cost) - - -def provider_stated_usage_cost(obj: object, usage_cost: object) -> object: - """What the provider itself put in usage.cost, looking past a bill litellm stamped there.""" - hidden_params: Final = get_hidden_params(obj) - if hidden_params is None or USAGE_COST_BEFORE_LITELLM_STAMP not in hidden_params: - return usage_cost - return hidden_params.get(USAGE_COST_BEFORE_LITELLM_STAMP) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ef47e2f0d9f..ace57a64aae 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -6,23 +6,15 @@ from typing import Final import httpx from pydantic import TypeAdapter, ValidationError -import litellm from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers -from litellm.litellm_core_utils.hidden_params import ( - USAGE_COST_BEFORE_LITELLM_STAMP, - get_hidden_params, - stamp_billable_usage_cost, -) from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.logging_utils import LiteLLMLoggingObject -from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ( EmbeddingResponse, HiddenParams, ModelResponse, TranscriptionResponse, - Usage, ) @@ -167,18 +159,16 @@ class ResponseMetadata: ## ADD OTHER HIDDEN PARAMS model_info: Final = kwargs.get("model_info", {}) or {} model_id: Final = model_info.get("id", None) - response_cost: Final = logging_obj.response_cost_calculator( - result=self.result, - litellm_model_name=model, - router_model_id=model_id, - cache_hit=cache_hit, - ) - self._stamp_billed_usage_cost(response_cost) new_params: Final = { "litellm_call_id": getattr(logging_obj, "litellm_call_id", None), "api_base": get_api_base(model=model or "", optional_params=kwargs), "model_id": model_id, - "response_cost": response_cost, + "response_cost": logging_obj.response_cost_calculator( + result=self.result, + litellm_model_name=model, + router_model_id=model_id, + cache_hit=cache_hit, + ), "additional_headers": process_response_headers( self._get_additional_headers_from_hidden_params() or {}, preserve_litellm_internal_headers=True, @@ -189,25 +179,6 @@ class ResponseMetadata: } self._update_hidden_params(new_params) - def _stamp_billed_usage_cost(self, response_cost: object) -> None: - """A usage.cost the upstream reported becomes what litellm billed, so a client and the spend log agree on a - priced deployment. A reply without one keeps its usage as the upstream sent it unless the operator asked for - the bill in every usage block with ``litellm.include_cost_in_streaming_usage``, the same switch that puts it - into streamed usage.""" - if not isinstance(self.result, (ModelResponse, ResponsesAPIResponse)): - return - usage: Final = getattr(self.result, "usage", None) - if not isinstance(usage, (Usage, ResponseAPIUsage)): - return - if getattr(usage, "cost", None) is None and not litellm.include_cost_in_streaming_usage: - return - if stamp_billable_usage_cost(self.result, usage, response_cost) is None: - return - stamped: Final = get_hidden_params(self.result) - if stamped is None or USAGE_COST_BEFORE_LITELLM_STAMP not in stamped: - return - self._update_hidden_params({USAGE_COST_BEFORE_LITELLM_STAMP: stamped[USAGE_COST_BEFORE_LITELLM_STAMP]}) - def _update_hidden_params(self, new_params: Mapping[str, object]) -> None: """ Update hidden params - handles when self._hidden_params is a dict or HiddenParams object diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index b71ac13684f..7cf024be2a1 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billable_usage_cost +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict @@ -1411,13 +1411,13 @@ def _set_stream_builder_response_cost( _set_hidden_response_cost(response, response_cost) -def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: +def _stamp_streaming_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - billed: Final = stamp_billable_usage_cost(response, usage, logging_obj.response_cost_calculator(result=response)) - if billed is None: + computed_cost: Final = logging_obj.response_cost_calculator(result=response) + if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: return - _set_hidden_response_cost(response, billed) + _set_hidden_response_cost(response, float(computed_cost)) def _field_of(value: object, name: str) -> object: @@ -1587,7 +1587,7 @@ def stream_chunk_builder( response, logging_obj, ) - _stamp_streaming_usage_cost(usage, response, logging_obj) + _stamp_streaming_response_cost(response, logging_obj) _set_stream_builder_response_cost( response, logging_obj, @@ -1775,7 +1775,7 @@ def stream_chunk_builder( ) break - _stamp_streaming_usage_cost(usage, response, logging_obj) + _stamp_streaming_response_cost(response, logging_obj) _set_stream_builder_response_cost( response, logging_obj, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 3b9c17baa02..c5843da28d9 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -130,6 +130,7 @@ from litellm.proxy.route_llm_request import ( route_request, ) from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id +from litellm.proxy.usage_cost_egress import with_client_usage_cost from litellm.proxy.utils import ( # noqa: F401 # legacy module exports ProxyLogging, _check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -273,6 +274,7 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( ModelResponse, ModelResponseStream, @@ -3325,7 +3327,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, ) - return response + return with_client_usage_cost(response, response_cost_for_headers) @staticmethod async def _record_container_owners_from_responses_if_needed( @@ -4710,6 +4712,7 @@ class ProxyBaseLLMRequestProcessing: completion_tokens_details=usage.get("completion_tokens_details"), prompt_tokens_details=usage.get("prompt_tokens_details"), server_tool_use=usage.get("server_tool_use"), + cost=usage.get("cost"), ) @staticmethod @@ -4724,7 +4727,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _completion_cost_or_none( - model_response: ModelResponse, model_name: str, service_tier: str | None + model_response: ModelResponse | ResponsesAPIResponse, model_name: str, service_tier: str | None ) -> float | None: try: return litellm.completion_cost( @@ -4735,7 +4738,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _logging_obj_cost_or_none( - model_response: ModelResponse, litellm_logging_obj: LiteLLMLoggingObj + model_response: ModelResponse | ResponsesAPIResponse, litellm_logging_obj: LiteLLMLoggingObj ) -> float | None: # Pricing a frame stamps cost_breakdown and, on failure, the cost-failure debug key onto # the live logging object. The pass-through handlers never recompute either one, so a @@ -4762,7 +4765,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _streamed_usage_cost( - model_response: ModelResponse, + model_response: ModelResponse | ResponsesAPIResponse, model_name: str, service_tier: str | None, litellm_logging_obj: LiteLLMLoggingObj | None, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bc79becf7bb..5c7388f7861 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -922,6 +922,7 @@ from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import ( router as user_banner_endpoints_router, ) +from litellm.proxy.usage_cost_egress import Pricer, with_client_stream_usage_cost from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports PrismaClient, ProxyLogging, @@ -10134,6 +10135,22 @@ def _fast_serialize_simple_model_response_stream( return orjson.dumps(payload) +def _stream_usage_pricer(request_data: Mapping[str, object]) -> Pricer: + """Bills a usage-bearing stream event the way the request's logging object bills the spend row.""" + logging_obj: Final = request_data.get("litellm_logging_obj") + model_name: Final = str(request_data.get("model", "")) + + def price(reply: ModelResponse | ResponsesAPIResponse) -> float | None: + return ProxyBaseLLMRequestProcessing._streamed_usage_cost( # pyright: ignore[reportPrivateUsage] # the proxy's one stream pricer + reply, + model_name, + None, + logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + ) + + return price + + def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: if isinstance(chunk, ModelResponseStream): serialized_chunk: Final = _fast_serialize_simple_model_response_stream(chunk) @@ -10472,6 +10489,7 @@ async def async_data_generator( needs_per_chunk_hook: Final = proxy_logging_obj.needs_per_chunk_streaming_hook() is_raw_sse_stream: Final = bool(request_data.get("_litellm_raw_sse_stream")) strip_stream_usage: Final = bool(request_data.get("_litellm_strip_stream_usage")) + price_stream_usage: Final = _stream_usage_pricer(request_data) raw_sse_buffer = "" # A stream can start on a deployment with keepalive off and fall back @@ -10558,7 +10576,7 @@ async def async_data_generator( error_state.observe_chunk(chunk) raw_passthrough = False if isinstance(chunk, BaseModel): - chunk = _serialize_streaming_chunk(chunk) + chunk = _serialize_streaming_chunk(with_client_stream_usage_cost(chunk, price_stream_usage)) elif isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="replace") if is_raw_sse_stream: diff --git a/litellm/proxy/usage_cost_egress.py b/litellm/proxy/usage_cost_egress.py new file mode 100644 index 00000000000..820fce85974 --- /dev/null +++ b/litellm/proxy/usage_cost_egress.py @@ -0,0 +1,77 @@ +"""What a client reads in ``usage.cost``: litellm's bill, written once on the way out of the proxy, onto a copy. + +Inside the pipeline ``usage.cost`` stays what the upstream reported, so pricing a reply again (an MCP round, a +logging copy, a stream rebuilt from its chunks) never reads litellm's own bill back as a provider report. +""" + +from __future__ import annotations + +import math +from collections.abc import Callable +from typing import Final, TypeVar + +from pydantic import BaseModel + +import litellm +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, ModelResponseStream, Usage + +Pricer = Callable[[ModelResponse | ResponsesAPIResponse], float | None] +"""Prices a reply the way the request's logging object bills it (deployment rates, discount, margin).""" + +_RESPONSES_USAGE_EVENTS: Final = frozenset({"response.completed", "response.incomplete"}) +_ReplyT = TypeVar("_ReplyT", bound=BaseModel) + + +def billable(cost: object) -> float | None: + if isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0: + return None + return float(cost) + + +def client_usage_cost(upstream_cost: object, bill: object, *, wanted: bool = False) -> float | None: + """The bill, when the upstream put a number in usage.cost or the operator wants it in every usage block + (``litellm.include_cost_in_streaming_usage``). None leaves the usage as the upstream sent it.""" + reported: Final = isinstance(upstream_cost, (int, float)) and not isinstance(upstream_cost, bool) + if not reported and not wanted and not litellm.include_cost_in_streaming_usage: + return None + return billable(bill) + + +def _with_usage_cost(reply: _ReplyT, usage: Usage | ResponseAPIUsage, cost: float) -> _ReplyT: + return reply.model_copy(update={"usage": usage.model_copy(update={"cost": cost})}) + + +def with_client_usage_cost(response: object, bill: object) -> object: + """A plain chat or Responses reply as the client gets it: a copy with the bill in usage.cost, or the reply + itself when its usage stays as the upstream sent it.""" + if not isinstance(response, (ModelResponse, ResponsesAPIResponse)): + return response + usage: Final = getattr(response, "usage", None) + if not isinstance(usage, (Usage, ResponseAPIUsage)): + return response + cost: Final = client_usage_cost(getattr(usage, "cost", None), bill) + if cost is None: + return response + return _with_usage_cost(response, usage, cost) + + +def with_client_stream_usage_cost(chunk: BaseModel, price: Pricer) -> BaseModel: + """A streamed event as the client gets it. Only the usage-bearing events are priced: the chat chunk carrying + ``usage`` and the Responses ``response.completed`` / ``response.incomplete`` events, which carry the bill + whether or not the upstream reported one, as they did before the bill moved out here.""" + if isinstance(chunk, ModelResponseStream): + usage: Final = getattr(chunk, "usage", None) + if not isinstance(usage, Usage): + return chunk + cost: Final = client_usage_cost(getattr(usage, "cost", None), price(ModelResponse(usage=usage))) + return chunk if cost is None else _with_usage_cost(chunk, usage, cost) + if getattr(chunk, "type", None) not in _RESPONSES_USAGE_EVENTS: + return chunk + response: Final = getattr(chunk, "response", None) + if not isinstance(response, ResponsesAPIResponse) or not isinstance(response.usage, ResponseAPIUsage): + return chunk + response_cost: Final = client_usage_cost(getattr(response.usage, "cost", None), price(response), wanted=True) + if response_cost is None: + return chunk + return chunk.model_copy(update={"response": _with_usage_cost(response, response.usage, response_cost)}) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index b5ef9e95741..dc84a955d88 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -25,7 +25,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) -from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, stamp_responses_usage_cost +from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator, normalize_responses_usage from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( PART_UNION_TYPES, @@ -1480,7 +1480,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): } ) ) - stamp_responses_usage_cost(partial, self.litellm_logging_obj) + normalize_responses_usage(partial) return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) def _emit_terminal_response_event( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index ca6bbf0046c..9daf350baa7 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,7 +41,6 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base -from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, stamp_billable_usage_cost from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -586,7 +585,7 @@ class BaseResponsesAPIStreamingIterator: else openai_responses_api_chunk.model_copy(update={"response": _billed_response}) ) self.completed_response = _terminal_chunk - stamp_responses_usage_cost(_billed_response, self.logging_obj) + normalize_responses_usage(_billed_response) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() @@ -676,17 +675,10 @@ class BaseResponsesAPIStreamingIterator: headers: Final[Mapping[str, object]] = processed if isinstance(processed, Mapping) else EMPTY_MAPPING raw_headers: Final[Mapping[str, object]] = raw if isinstance(raw, Mapping) else EMPTY_MAPPING # rebuild by value and let existing keys win: sharing the source dicts would alias what the proxy - # splats into the client's HTTP headers, and copying non-header keys would carry response_cost. - # The provider's own usage.cost travels along, or the stamped bill would be priced as a provider report - stamp: Final[Mapping[str, object]] = ( - {USAGE_COST_BEFORE_LITELLM_STAMP: source[USAGE_COST_BEFORE_LITELLM_STAMP]} - if USAGE_COST_BEFORE_LITELLM_STAMP in source - else EMPTY_MAPPING - ) + # splats into the client's HTTP headers, and copying non-header keys would carry response_cost target.hidden_params = { "additional_headers": {**headers}, "headers": {**raw_headers}, - **stamp, **existing, } @@ -1077,7 +1069,7 @@ class BaseResponsesAPIStreamingIterator: "output": _partial_output_items(self._generated_content), } ) - stamp_responses_usage_cost(partial, self.logging_obj) + normalize_responses_usage(partial) return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None: @@ -1749,20 +1741,14 @@ def _estimate_usage_safely( return None -def stamp_responses_usage_cost( - response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None -) -> None: - if response_obj is None or logging_obj is None: +def normalize_responses_usage(response_obj: ResponsesAPIResponse | None) -> None: + """A terminal response built off an untyped dict carries its usage as a dict, logging and the proxy read a model.""" + if response_obj is None: return usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return - response_obj.usage = usage_obj # rebind-ok: the stamped cost has to ride on the response the client receives - try: - cost: Final[float | None] = logging_obj.response_cost_calculator(result=response_obj) - except Exception: - return - stamp_billable_usage_cost(response_obj, usage_obj, cost) + response_obj.usage = usage_obj # rebind-ok: the typed usage has to ride on the response the client receives def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: @@ -1796,7 +1782,7 @@ def build_synthetic_response_events( chunk_size: int, ) -> list[ResponsesAPIStreamingResponse]: openai_types: Final = get_openai_response_types() - stamp_responses_usage_cost(transformed, logging_obj) + normalize_responses_usage(transformed) events: Final[list[ResponsesAPIStreamingResponse]] = [ _build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed), diff --git a/tests/unit/proxy/test_usage_cost_egress.py b/tests/unit/proxy/test_usage_cost_egress.py new file mode 100644 index 00000000000..3c62ddb9308 --- /dev/null +++ b/tests/unit/proxy/test_usage_cost_egress.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +import litellm +from litellm.proxy.usage_cost_egress import with_client_stream_usage_cost, with_client_usage_cost +from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from litellm.types.utils import Choices, Message, ModelResponse, ModelResponseStream, StreamingChoices, Usage + +UPSTREAM_COST: Final = 0.037 +OPERATOR_BILL: Final = 100 * 2e-6 + 20 * 6e-6 + + +def _chat_reply(upstream_cost: float | None) -> ModelResponse: + reply: Final = ModelResponse( + id="chatcmpl-egress", + model="omni-model", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=upstream_cost), + ) + reply._hidden_params["response_cost"] = OPERATOR_BILL + return reply + + +def _responses_reply(upstream_cost: float | None) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_egress", + created_at=1, + status="completed", + model="omni-model", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=upstream_cost), + ) + + +def _usage_chunk(upstream_cost: float | None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-egress", + model="omni-model", + choices=[], + usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=upstream_cost), + ) + + +def _bill(_reply: object) -> float | None: + return OPERATOR_BILL + + +def _unpriceable(_reply: object) -> float | None: + return None + + +@pytest.mark.parametrize("reply", [_chat_reply(UPSTREAM_COST), _responses_reply(UPSTREAM_COST)]) +def test_a_reply_whose_upstream_reported_a_cost_tells_the_client_what_litellm_billed(reply: object) -> None: + billed: Final = with_client_usage_cost(reply, OPERATOR_BILL) + + assert billed is not reply, "the pipeline's own object keeps the upstream figure" + assert getattr(getattr(billed, "usage", None), "cost", None) == pytest.approx(OPERATOR_BILL) + assert getattr(getattr(reply, "usage", None), "cost", None) == UPSTREAM_COST + + +def test_the_billed_copy_of_a_chat_reply_keeps_its_hidden_params() -> None: + reply: Final = _chat_reply(UPSTREAM_COST) + + billed: Final = with_client_usage_cost(reply, OPERATOR_BILL) + + assert isinstance(billed, ModelResponse) + assert billed._hidden_params["response_cost"] == pytest.approx(OPERATOR_BILL) + assert billed.model_dump()["usage"]["cost"] == pytest.approx(OPERATOR_BILL) + + +@pytest.mark.parametrize("reply", [_chat_reply(None), _responses_reply(None)]) +def test_a_reply_without_an_upstream_cost_goes_out_as_the_upstream_sent_it( + monkeypatch: pytest.MonkeyPatch, reply: object +) -> None: + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) + + assert with_client_usage_cost(reply, OPERATOR_BILL) is reply + + +@pytest.mark.parametrize("reply", [_chat_reply(None), _responses_reply(None)]) +def test_include_cost_in_streaming_usage_puts_the_bill_on_a_reply_without_an_upstream_cost( + monkeypatch: pytest.MonkeyPatch, reply: object +) -> None: + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) + + billed: Final = with_client_usage_cost(reply, OPERATOR_BILL) + + assert getattr(getattr(billed, "usage", None), "cost", None) == pytest.approx(OPERATOR_BILL) + + +def test_a_zero_bill_replaces_the_upstream_cost() -> None: + billed: Final = with_client_usage_cost(_chat_reply(UPSTREAM_COST), 0.0) + + assert getattr(getattr(billed, "usage", None), "cost", None) == 0.0 + + +@pytest.mark.parametrize("bill", [None, -0.001, float("nan"), float("inf"), True, "0.1"]) +def test_an_unpriceable_reply_keeps_the_upstream_cost(bill: object) -> None: + reply: Final = _chat_reply(UPSTREAM_COST) + + assert with_client_usage_cost(reply, bill) is reply + assert reply.usage is not None and reply.usage.cost == UPSTREAM_COST + + +def test_a_reply_that_is_neither_chat_nor_responses_passes_through() -> None: + reply: Final = {"usage": {"cost": UPSTREAM_COST}} + + assert with_client_usage_cost(reply, OPERATOR_BILL) is reply + + +def test_the_streamed_usage_chunk_tells_the_client_what_litellm_billed() -> None: + chunk: Final = _usage_chunk(UPSTREAM_COST) + + billed: Final = with_client_stream_usage_cost(chunk, _bill) + + assert isinstance(billed, ModelResponseStream) and billed is not chunk + assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) + assert chunk.usage is not None and chunk.usage.cost == UPSTREAM_COST + + +def test_a_content_chunk_is_not_priced() -> None: + chunk: Final = ModelResponseStream( + id="chatcmpl-egress", model="omni-model", choices=[StreamingChoices(index=0, delta={"content": "hi"})] + ) + priced: list[object] = [] + + def price(reply: object) -> float | None: + priced.append(reply) + return OPERATOR_BILL + + assert with_client_stream_usage_cost(chunk, price) is chunk + assert priced == [] + + +def test_a_streamed_usage_chunk_without_an_upstream_cost_goes_out_as_sent(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) + chunk: Final = _usage_chunk(None) + + assert with_client_stream_usage_cost(chunk, _bill) is chunk + + +def test_include_cost_in_streaming_usage_puts_the_bill_on_the_streamed_usage_chunk( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) + + billed: Final = with_client_stream_usage_cost(_usage_chunk(None), _bill) + + assert isinstance(billed, ModelResponseStream) + assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) + + +def test_the_stream_pricer_sees_the_upstream_cost_so_the_bill_matches_the_spend_row() -> None: + seen: list[object] = [] + + def price(reply: object) -> float | None: + seen.append(getattr(getattr(reply, "usage", None), "cost", None)) + return OPERATOR_BILL + + with_client_stream_usage_cost(_usage_chunk(UPSTREAM_COST), price) + + assert seen == [UPSTREAM_COST] + + +@pytest.mark.parametrize("upstream_cost", [UPSTREAM_COST, None]) +@pytest.mark.parametrize( + "event_type", [ResponsesAPIStreamEvents.RESPONSE_COMPLETED, ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] +) +def test_a_terminal_responses_event_carries_the_bill_with_or_without_an_upstream_cost( + monkeypatch: pytest.MonkeyPatch, upstream_cost: float | None, event_type: ResponsesAPIStreamEvents +) -> None: + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) + event_cls: Final = ( + ResponseCompletedEvent if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED else ResponseIncompleteEvent + ) + event: Final = event_cls(type=event_type, response=_responses_reply(upstream_cost)) + + billed: Final = with_client_stream_usage_cost(event, _bill) + + assert billed is not event + assert getattr(billed, "response").usage.cost == pytest.approx(OPERATOR_BILL) + assert event.response.usage is not None and event.response.usage.cost == upstream_cost + + +def test_an_unpriceable_terminal_responses_event_goes_out_as_sent() -> None: + event: Final = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_reply(None) + ) + + assert with_client_stream_usage_cost(event, _unpriceable) is event diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 8d624a186ad..bd9122a927c 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -3,15 +3,18 @@ completion_start_time on the first chunk so downstream TTFT consumers (Prometheus, OTEL, SpendLogs completionStartTime) do not fall back to completion_start_time = end_time.""" -import asyncio, importlib +import asyncio import functools import gc +import importlib import json import threading import weakref from collections.abc import Callable, Mapping +from contextlib import suppress from datetime import datetime -from typing import Final, Optional +from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx @@ -24,7 +27,8 @@ from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.responses.streaming_iterator import( +from litellm.responses import streaming_iterator as streaming_module +from litellm.responses.streaming_iterator import ( CachedResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, ResponsesAPIStreamingIterator, @@ -38,15 +42,12 @@ from litellm.types.llms.openai import ( ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) -from contextlib import suppress -from litellm.responses import streaming_iterator as streaming_module from litellm.types.utils import CallTypes from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome -from types import SimpleNamespace def _sse_event(payload: dict) -> bytes: - return f"data: {json.dumps(payload)}\n\n".encode("utf-8") + return f"data: {json.dumps(payload)}\n\n".encode() def _mock_config() -> Mock: @@ -80,7 +81,7 @@ def _make_iterator( *, sse_events: list[bytes], logging_obj: LiteLLMLoggingObj, - trailing_error: Optional[Exception] = None, + trailing_error: Exception | None = None, config: Mock | None = None, request_data: dict | None = None, ) -> ResponsesAPIStreamingIterator: @@ -109,7 +110,7 @@ def _make_sync_iterator( *, sse_events: list[bytes], logging_obj: LiteLLMLoggingObj, - trailing_error: Optional[Exception] = None, + trailing_error: Exception | None = None, ) -> SyncResponsesAPIStreamingIterator: def iter_bytes(): for evt in sse_events: @@ -489,7 +490,6 @@ class _LoopRecordingLogger(CustomLogger): self.hook_loop = asyncio.get_running_loop() await asyncio.sleep(0.05) self.hook_finished = True - return None class _SyncOnlyRecordingLogger(CustomLogger): @@ -838,51 +838,6 @@ def _responses_api_response_with_usage() -> ResponsesAPIResponse: ) -def test_stamp_responses_usage_cost_stamps_computed_cost(): - from litellm.responses.streaming_iterator import stamp_responses_usage_cost - - response = _responses_api_response_with_usage() - logging_obj = Mock(spec=LiteLLMLoggingObj) - logging_obj.response_cost_calculator.return_value = 0.000704 - - stamp_responses_usage_cost(response, logging_obj) - - assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_called_once_with(result=response) - - -def test_stamp_responses_usage_cost_puts_the_bill_over_the_provider_reported_cost(): - from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP, provider_stated_usage_cost - from litellm.responses.streaming_iterator import stamp_responses_usage_cost - - response = _responses_api_response_with_usage() - setattr(response.usage, "cost", 0.5) - logging_obj = Mock(spec=LiteLLMLoggingObj) - logging_obj.response_cost_calculator.return_value = 0.00032 - - stamp_responses_usage_cost(response, logging_obj) - - assert getattr(response.usage, "cost", None) == pytest.approx(0.00032) - assert provider_stated_usage_cost(response, response.usage.cost) == pytest.approx(0.5) - assert response._hidden_params[USAGE_COST_BEFORE_LITELLM_STAMP] == pytest.approx(0.5) - - -def test_stamp_responses_usage_cost_stamps_a_zero_bill_only_over_a_provider_reported_cost(): - from litellm.responses.streaming_iterator import stamp_responses_usage_cost - - unpriced = _responses_api_response_with_usage() - reported = _responses_api_response_with_usage() - setattr(reported.usage, "cost", 0.5) - logging_obj = Mock(spec=LiteLLMLoggingObj) - logging_obj.response_cost_calculator.return_value = 0.0 - - stamp_responses_usage_cost(unpriced, logging_obj) - stamp_responses_usage_cost(reported, logging_obj) - - assert getattr(unpriced.usage, "cost", None) is None - assert getattr(reported.usage, "cost", None) == 0.0 - - def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: return ResponsesAPIResponse.model_construct( id="resp_lit7391", @@ -896,8 +851,9 @@ def _unvalidated_response_with_dict_usage(usage: dict) -> ResponsesAPIResponse: ) -def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): - from litellm.responses.streaming_iterator import stamp_responses_usage_cost +def test_normalize_responses_usage_types_a_dict_usage_and_keeps_the_provider_cost(): + from litellm.responses.streaming_iterator import normalize_responses_usage + response = _unvalidated_response_with_dict_usage( { "input_tokens": 29, @@ -907,37 +863,20 @@ def test_stamp_responses_usage_cost_keeps_provider_cost_from_dict_usage(): "cost": {"currency": "USD", "input_cost": 0, "output_cost": 3e-05, "total_cost": 3e-05}, } ) - logging_obj = Mock(spec=LiteLLMLoggingObj) - logging_obj.response_cost_calculator.return_value = 3e-05 - stamp_responses_usage_cost(response, logging_obj) + normalize_responses_usage(response) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) assert response.usage.output_tokens_details.reasoning_tokens == 117 -def test_stamp_responses_usage_cost_computes_cost_for_dict_usage_without_cost(): - from litellm.responses.streaming_iterator import stamp_responses_usage_cost - response = _unvalidated_response_with_dict_usage({"input_tokens": 29, "output_tokens": 120, "total_tokens": 149}) - logging_obj = Mock(spec=LiteLLMLoggingObj) - logging_obj.response_cost_calculator.return_value = 0.000704 - - stamp_responses_usage_cost(response, logging_obj) - - assert isinstance(response.usage, ResponseAPIUsage) - assert response.usage.cost == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_called_once_with(result=response) - - -def test_stamp_responses_usage_cost_survives_calculator_failure(): - from litellm.responses.streaming_iterator import stamp_responses_usage_cost +def test_normalize_responses_usage_never_writes_a_cost_the_provider_did_not_report(): + from litellm.responses.streaming_iterator import normalize_responses_usage response = _responses_api_response_with_usage() - logging_obj = Mock(spec=LiteLLMLoggingObj) - logging_obj.response_cost_calculator.side_effect = RuntimeError("cost map unavailable") - stamp_responses_usage_cost(response, logging_obj) + normalize_responses_usage(response) assert getattr(response.usage, "cost", None) is None @@ -959,7 +898,7 @@ def _capture_dispatch(logged: list): return _dispatch -def _headers_config(*, transform_hidden_params: Optional[dict] = None) -> Mock: +def _headers_config(*, transform_hidden_params: dict | None = None) -> Mock: """Config whose completed event carries a real ResponsesAPIResponse, so the logging copy performs a genuine model_dump/model_validate round trip.""" mock_config = Mock(spec=BaseResponsesAPIConfig) @@ -1274,10 +1213,10 @@ async def test_completed_event_without_usage_counts_tool_input_deltas(tool_delta @pytest.mark.asyncio -async def test_completed_event_with_a_dict_response_is_typed_and_billed(): +async def test_completed_event_with_a_dict_response_is_typed(): """transform_streaming_response can model_construct a terminal event whose response stays a plain dict; the iterator must type it so the estimated - usage reaches the cost stamping path.""" + usage reaches logging and the proxy egress as a model.""" dict_response: Final = { "id": "resp_dict", "model": "gpt-4o-mini", @@ -1319,8 +1258,7 @@ async def test_completed_event_with_a_dict_response_is_typed_and_billed(): assert usage is not None assert usage.input_tokens > 0 assert usage.output_tokens > 0 - assert usage.cost == pytest.approx(0.000704) - logging_obj.response_cost_calculator.assert_any_call(result=completed_response) + assert getattr(usage, "cost", None) is None, "the bill goes on at the proxy egress, never on the event logging prices" def test_billed_terminal_response_keeps_a_response_that_already_has_usage(): @@ -1390,6 +1328,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def setup_and_teardown(): """ @@ -1407,6 +1346,7 @@ def setup_and_teardown(): loop.close() asyncio.set_event_loop(None) + class _FakeLoggingObj: def __init__(self): self.success_calls = 0 @@ -1447,10 +1387,12 @@ class _FakeLoggingObj: self.completion_start_time = completion_start_time self.model_call_details["completion_start_time"] = completion_start_time + async def _await_tasks_spawned_since(existing: set["asyncio.Task[object]"]) -> None: spawned: Final = asyncio.all_tasks() - existing await asyncio.wait_for(asyncio.gather(*spawned, return_exceptions=True), timeout=5) + def _make_completed_response(response_id: str = "resp_test") -> ResponseCompletedEvent: return ResponseCompletedEvent( type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, @@ -1478,6 +1420,7 @@ def _make_completed_response(response_id: str = "resp_test") -> ResponseComplete ), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_log_background_task_failure_logs_task_exceptions(monkeypatch): @@ -1500,6 +1443,7 @@ async def test_log_background_task_failure_logs_task_exceptions(monkeypatch): task.exception(), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch): @@ -1515,6 +1459,7 @@ async def test_log_background_task_failure_ignores_cancelled_tasks(monkeypatch): error_logger.assert_not_called() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_content_part_done_event_supports_refusal_and_reasoning_text(): refusal_event = streaming_module._build_content_part_done_event( @@ -1542,6 +1487,7 @@ def test_content_part_done_event_supports_refusal_and_reasoning_text(): assert reasoning_event.part.reasoning == "because" assert unsupported_event is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_dump_response_object_handles_model_and_unknown_values(): response = ResponsesAPIResponse( @@ -1557,6 +1503,7 @@ def test_dump_response_object_handles_model_and_unknown_values(): assert streaming_module._dump_response_object({"type": "message"}) == {"type": "message"} assert streaming_module._dump_response_object(object()) == {} + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_triggers_hooks(monkeypatch): @@ -1613,6 +1560,7 @@ async def test_responses_streaming_triggers_hooks(monkeypatch): assert seen["request_data"].get("litellm_params") is not None assert seen["call_type"] == CallTypes.responses + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypatch): @@ -1653,6 +1601,7 @@ async def test_responses_streaming_calls_post_streaming_deployment_hook(monkeypa # reset callbacks litellm.callbacks = original_callbacks + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_failure_triggers_failure_handlers(): @@ -1682,6 +1631,7 @@ async def test_responses_streaming_failure_triggers_failure_handlers(): assert logging_obj.failure_calls >= 1 assert logging_obj.async_failure_calls >= 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_requires_provider_config(): iterator = ResponsesAPIStreamingIterator( @@ -1696,6 +1646,7 @@ def test_process_chunk_requires_provider_config(): with pytest.raises(ValueError, match="responses_api_provider_config is required"): iterator._process_chunk(json.dumps({"type": "response.completed"})) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_wraps_encrypted_content_with_model_id(): openai_types = streaming_module._get_openai_response_types() @@ -1730,8 +1681,9 @@ def test_process_chunk_wraps_encrypted_content_with_model_id(): assert event.item.encrypted_content.startswith("litellm_enc:") assert event.item.encrypted_content.endswith(";ciphertext") + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") -def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch): +def test_process_chunk_completed_response_updates_id_and_leaves_usage_cost_alone(monkeypatch): original_include_cost = litellm.include_cost_in_streaming_usage litellm.include_cost_in_streaming_usage = True openai_types = streaming_module._get_openai_response_types() @@ -1780,9 +1732,10 @@ def test_process_chunk_completed_response_updates_id_and_usage_cost(monkeypatch) assert iterator.completed_response is event assert event.response.id != "resp_live" assert event.response.id.startswith("resp_") - assert event.response.usage.cost == 1.23 + assert event.response.usage.cost is None, "the bill goes on at the proxy egress, never on the event logging prices" completion_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch): openai_types = streaming_module._get_openai_response_types() @@ -1818,6 +1771,7 @@ def test_process_chunk_failed_response_triggers_failure_logging(monkeypatch): assert iterator.completed_response is event failure_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_handle_logging_failed_response_uses_response_error_message(): @@ -1850,6 +1804,7 @@ async def test_handle_logging_failed_response_uses_response_error_message(): assert logging_obj.failure_calls == 1 assert logging_obj.async_failure_calls == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload(): class _NoopConfig: @@ -1868,6 +1823,7 @@ def test_process_chunk_returns_none_for_invalid_json_and_non_dict_payload(): assert iterator._process_chunk("not-json") is None assert iterator._process_chunk(json.dumps(["not", "a", "dict"])) is None + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch): original_include_cost = litellm.include_cost_in_streaming_usage @@ -1915,6 +1871,7 @@ def test_process_chunk_cost_annotation_failure_is_nonfatal(monkeypatch): assert event.response.usage.cost is None completion_handler.assert_called_once() + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_get_completed_response_object_accepts_direct_response(): logging_obj = _FakeLoggingObj() @@ -1931,6 +1888,7 @@ def test_get_completed_response_object_accepts_direct_response(): assert iterator._get_completed_response_object() is direct_response + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_responses_streaming_completed_event_persists_async_cache(): @@ -1979,6 +1937,7 @@ async def test_responses_streaming_completed_event_persists_async_cache(): assert json.loads(litellm.cache.async_add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_responses_streaming_completed_event_persists_sync_cache(): logging_obj = _FakeLoggingObj() @@ -2024,6 +1983,7 @@ def test_responses_streaming_completed_event_persists_sync_cache(): assert json.loads(litellm.cache.add_cache.call_args.args[0])["id"] == iterator.completed_response.response.id litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_log_completed_response_sync_direct_path(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2065,6 +2025,7 @@ def test_log_completed_response_sync_direct_path(monkeypatch): assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch): class _BadSerializableResponse: @@ -2094,6 +2055,7 @@ def test_log_completed_response_falls_back_when_model_validate_fails(monkeypatch assert logging_obj.success_calls == 1 assert logging_obj.async_success_calls == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize( "scenario", @@ -2158,6 +2120,7 @@ def test_persist_completed_response_to_cache_guard_branches(monkeypatch, scenari expected_cached_flag = scenario == "already_cached" assert iterator._completed_response_cached is expected_cached_flag + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_build_synthetic_response_events_covers_annotations_function_calls_and_refusals(): original_include_cost = litellm.include_cost_in_streaming_usage @@ -2216,6 +2179,7 @@ def test_build_synthetic_response_events_covers_annotations_function_calls_and_r assert "response.function_call_arguments.done" in event_types assert event_types[-1] == "response.completed" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_mock_responses_streaming_iterator_async_iteration_logs_completion( @@ -2266,6 +2230,7 @@ async def test_mock_responses_streaming_iterator_async_iteration_logs_completion assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2311,6 +2276,7 @@ def test_mock_responses_streaming_iterator_sync_iteration_logs_completion(monkey assert hook_calls["post_call"] == 1 assert hook_calls["metadata"] == 1 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.asyncio async def test_cached_responses_stream_async_hit_triggers_success_callbacks( @@ -2370,6 +2336,7 @@ async def test_cached_responses_stream_async_hit_triggers_success_callbacks( litellm.cache.add_cache.assert_not_called() litellm.cache = original_cache + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_cached_responses_stream_sync_hit_triggers_success_callbacks(monkeypatch): hook_calls = {"post_call": 0, "metadata": 0} @@ -2468,7 +2435,9 @@ def test_faked_stream_bridged_to_chat_sends_tool_arguments_once(): tool_call_deltas: Final = tuple( tool_call for chunk in chunks - for tool_call in (chunk.choices[0].delta.tool_calls or ()) # comprehension-ok: flattening each chunk's tool calls + for tool_call in ( + chunk.choices[0].delta.tool_calls or () + ) # comprehension-ok: flattening each chunk's tool calls ) joined_arguments: Final = "".join(tool_call.function.arguments or "" for tool_call in tool_call_deltas) joined_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks) @@ -2524,7 +2493,6 @@ def test_faked_stream_opens_items_in_progress_and_empty_like_a_real_stream(): assert [part.text for part in added_parts] == [""] - def _answer_with_reasoning_refusal_and_custom_tool_call() -> ResponsesAPIResponse: return ResponsesAPIResponse( id="resp_synthetic", @@ -2658,7 +2626,9 @@ async def test_shutdown_drain_waits_for_a_pending_logging_task(): @pytest.mark.asyncio async def test_shutdown_drain_gives_up_on_a_stuck_logging_task_at_its_timeout(): - stuck: Final = streaming_module.spawn_logging_task(asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck") + stuck: Final = streaming_module.spawn_logging_task( + asyncio.get_running_loop(), asyncio.Event().wait(), task_name="stuck" + ) await asyncio.wait_for(streaming_module.drain_logging_tasks(timeout=0.05), timeout=1) @@ -2800,39 +2770,3 @@ async def test_websocket_turns_logging_survives_garbage_collection(terminal_even await _run_websocket_turn(logger, terminal_event) assert await _logging_tasks_left_after_garbage_collection(logger) == 1 - - -def test_logging_copy_keeps_the_provider_usage_cost_behind_a_stamped_bill() -> None: - from litellm.litellm_core_utils.hidden_params import USAGE_COST_BEFORE_LITELLM_STAMP - from litellm.responses.streaming_iterator import _logging_copy - - logging_obj = LiteLLMLoggingObj( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "hi"}], - stream=True, - call_type="aresponses", - start_time=datetime.now(), - litellm_call_id="stamped-copy", - function_id="stamped-copy", - ) - iterator = _make_iterator(sse_events=[], logging_obj=logging_obj) - billed = ResponsesAPIResponse( - id="resp_stamped", - created_at=1, - status="completed", - model="gpt-4o-mini", - object="response", - output=[], - usage={"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.63}, - ) - billed.hidden_params = {USAGE_COST_BEFORE_LITELLM_STAMP: 0.42, "response_cost": 0.63} - iterator.completed_response = ResponseCompletedEvent( - type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=billed - ) - logging_response = _logging_copy(iterator.completed_response) - - iterator._restore_provider_response_headers(logging_response) - - copied = logging_response.response.hidden_params - assert copied[USAGE_COST_BEFORE_LITELLM_STAMP] == 0.42 - assert "response_cost" not in copied diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index b6ff6cadb8d..78480dc4f31 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1,48 +1,75 @@ -import json import asyncio import datetime +import json +import os +import threading import time from collections.abc import Iterator, Mapping from types import MappingProxyType, SimpleNamespace -from typing import Final, Literal, cast +from typing import Final, cast +from unittest.mock import MagicMock, patch import httpx import pytest import respx -from openai.types.completion_usage import CompletionUsage from pydantic import BaseModel import litellm from litellm import TranscriptionResponse, model_cost from litellm.constants import REALTIME_METERED_USAGE_KEY -from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator -from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details +from litellm.cost_calculator import ( + BaseTokenUsageProcessor, + RealtimeAPITokenUsageProcessor, + ResponsesWebSocketTokenUsageProcessor, + batch_cost_calculator, + completion_cost, + cost_per_token, + handle_realtime_stream_cost_calculation, + response_cost_calculator, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.fallback_generalizations import ( get_fallback_generalization_rules, set_fallback_generalizations, ) -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.llm_response_utils.response_metadata import update_response_metadata -from litellm.responses.streaming_iterator import stamp_responses_usage_cost +from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, ) +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.fireworks_ai.cost_calculator import get_base_model_for_pricing -from litellm.llms.together_ai.cost_calculator import get_model_params_and_category from litellm.types.llms.base import CachedTokensDetails -from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.llms.openai import ( + HttpxBinaryResponseContent, + OpenAIRealtimeStreamList, + ResponseAPIUsage, + ResponsesAPIResponse, +) from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse -from litellm.types.utils import CacheCreationTokenDetails, CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import ( + CacheCreationTokenDetails, + CallTypes, + Choices, + CompletionTokensDetailsWrapper, + EmbeddingResponse, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, + LiteLLMRealtimeStreamLoggingObject, + Message, + ModelInfo, + ModelResponse, + PromptTokensDetails, + PromptTokensDetailsWrapper, + Usage, +) from litellm.types.videos.main import VideoObject from litellm.utils import supports_prompt_caching -import os -import threading -from unittest.mock import MagicMock, patch class ResponseCostCapture(CustomLogger): @@ -518,9 +545,7 @@ def test_completion_cost_prices_base_model_under_its_own_provider( ) info: Final = litellm.get_model_info("claude-sonnet-4-5", custom_llm_provider="anthropic") - assert cost == pytest.approx( - 1000 * info["input_cost_per_token"] + 1000 * info["output_cost_per_token"] - ) + assert cost == pytest.approx(1000 * info["input_cost_per_token"] + 1000 * info["output_cost_per_token"]) def test_completion_cost_strips_dated_azure_snapshot_model(_local_model_cost_map: None) -> None: @@ -1615,9 +1640,7 @@ def test_per_query_priced_rerank_deployment_completion_cost_is_nonzero(): "vertex_ai/semantic-ranker-fast-004", ), ) -def test_vertex_semantic_ranker_rerank_completion_cost_is_nonzero( - model: str, _local_model_cost_map: None -) -> None: +def test_vertex_semantic_ranker_rerank_completion_cost_is_nonzero(model: str, _local_model_cost_map: None) -> None: response: Final = RerankResponse( id="vertex_ai_rerank_pricing_test", results=[], @@ -3373,9 +3396,7 @@ def test_gemini_cache_tokens_details_no_negative_values(): assert usage.prompt_tokens_details.text_tokens == 9402 assert usage.prompt_tokens_details.image_tokens == 258 assert usage.prompt_tokens_details.cached_tokens == 9651 - assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails( - text_tokens=9393, image_tokens=258 - ) + assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=9393, image_tokens=258) assert prompt_details["text_tokens"] == 9 assert prompt_details["image_tokens"] == 0 assert prompt_details["text_tokens"] >= 0 @@ -6876,23 +6897,17 @@ def test_cost_ft_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost( - completion_response=resp, custom_llm_provider="openai" - ) + cost = litellm.completion_cost(completion_response=resp, custom_llm_provider="openai") print("\n Calculated Cost for ft:gpt-3.5", cost) input_cost = model_cost["ft:gpt-3.5-turbo"]["input_cost_per_token"] output_cost = model_cost["ft:gpt-3.5-turbo"]["output_cost_per_token"] print(input_cost, output_cost) - expected_cost = (input_cost * resp.usage.prompt_tokens) + ( - output_cost * resp.usage.completion_tokens - ) + expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: print(f"Error: {e}") - pytest.fail( - f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}" - ) + pytest.fail(f"Cost Calc failed for ft:gpt-3.5. Expected {expected_cost}, Calculated cost {cost}") def test_cost_azure_gpt_35(): @@ -6918,28 +6933,22 @@ def test_cost_azure_gpt_35(): usage=Usage(prompt_tokens=21, completion_tokens=17, total_tokens=38), ) - cost = litellm.completion_cost( - completion_response=resp, model="azure/chatgpt-deployment-2" - ) + cost = litellm.completion_cost(completion_response=resp, model="azure/chatgpt-deployment-2") print("\n Calculated Cost for azure/gpt-3.5-turbo", cost) input_cost = model_cost["azure/gpt-35-turbo"]["input_cost_per_token"] output_cost = model_cost["azure/gpt-35-turbo"]["output_cost_per_token"] - expected_cost = (input_cost * resp.usage.prompt_tokens) + ( - output_cost * resp.usage.completion_tokens - ) + expected_cost = (input_cost * resp.usage.prompt_tokens) + (output_cost * resp.usage.completion_tokens) print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: - pytest.fail(f"Cost Calc failed for azure/gpt-3.5-turbo. {str(e)}") + pytest.fail(f"Cost Calc failed for azure/gpt-3.5-turbo. {e!s}") def test_cost_bedrock_pricing_actual_calls(): litellm.set_verbose = True model = "anthropic.claude-3-5-sonnet-20240620-v1:0" messages = [{"role": "user", "content": "Hey, how's it going?"}] - response = litellm.completion( - model=model, messages=messages, mock_response="hello cool one" - ) + response = litellm.completion(model=model, messages=messages, mock_response="hello cool one") print("response", response) cost = litellm.completion_cost( @@ -6970,8 +6979,7 @@ def test_whisper_openai(): print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] - * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -6991,15 +6999,12 @@ def test_whisper_azure(): _total_time_in_seconds = 3 setattr(transcription, "duration", _total_time_in_seconds) - cost = litellm.completion_cost( - model="azure/azure-whisper", completion_response=transcription - ) + cost = litellm.completion_cost(model="azure/azure-whisper", completion_response=transcription) print(f"cost: {cost}") print(f"whisper dict: {litellm.model_cost['whisper-1']}") expected_cost = round( - litellm.model_cost["whisper-1"]["output_cost_per_second"] - * _total_time_in_seconds, + litellm.model_cost["whisper-1"]["output_cost_per_second"] * _total_time_in_seconds, 5, ) assert round(cost, 5) == round(expected_cost, 5) @@ -7046,9 +7051,7 @@ def test_transcription_cost_falls_back_to_provider_billed_seconds( response_type="audio_transcription", ) - cost: Final = completion_cost( - completion_response=transcription, model=requested_model, call_type="atranscription" - ) + cost: Final = completion_cost(completion_response=transcription, model=requested_model, call_type="atranscription") assert cost == pytest.approx(0.01 * expected_seconds) @@ -7140,9 +7143,7 @@ def test_replicate_llama3_cost_tracking(): model="replicate/meta/meta-llama-3-8b-instruct", object="chat.completion", system_fingerprint=None, - usage=litellm.utils.Usage( - prompt_tokens=48, completion_tokens=31, total_tokens=79 - ), + usage=litellm.utils.Usage(prompt_tokens=48, completion_tokens=31, total_tokens=79), ) cost = litellm.completion_cost( completion_response=response, @@ -7152,20 +7153,14 @@ def test_replicate_llama3_cost_tracking(): print(f"cost: {cost}") cost = round(cost, 5) expected_cost = round( - litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ - "input_cost_per_token" - ] - * 48 - + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"][ - "output_cost_per_token" - ] - * 31, + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["input_cost_per_token"] * 48 + + litellm.model_cost["replicate/meta/meta-llama-3-8b-instruct"]["output_cost_per_token"] * 31, 5, ) assert cost == expected_cost -@pytest.mark.parametrize("is_streaming", [True, False]) # +@pytest.mark.parametrize("is_streaming", [True, False]) def test_groq_response_cost_tracking(is_streaming): from litellm.utils import ( CallTypes, @@ -7295,7 +7290,7 @@ def test_vertex_ai_completion_cost(): model_info = litellm.get_model_info(model="gemini-3.8-flash") - print("\nExpected model info:\n{}\n\n".format(model_info)) + print(f"\nExpected model info:\n{model_info}\n\n") expected_input_cost = prompt_tokens * model_info["input_cost_per_token"] @@ -7308,8 +7303,8 @@ def test_vertex_ai_completion_cost(): ) assert round(expected_input_cost, 6) == round(calculated_input_cost, 6) - print("expected_input_cost: {}".format(expected_input_cost)) - print("calculated_input_cost: {}".format(calculated_input_cost)) + print(f"expected_input_cost: {expected_input_cost}") + print(f"calculated_input_cost: {calculated_input_cost}") def test_vertex_ai_medlm_completion_cost(): @@ -7320,9 +7315,7 @@ def test_vertex_ai_medlm_completion_cost(): model = "vertex_ai/medlm-medium" messages = [{"role": "user", "content": "Test MedLM completion cost."}] - predictive_cost = completion_cost( - model=model, messages=messages, custom_llm_provider="vertex_ai" - ) + predictive_cost = completion_cost(model=model, messages=messages, custom_llm_provider="vertex_ai") assert predictive_cost > 0 model = "vertex_ai/medlm-large" @@ -7339,13 +7332,11 @@ def test_vertex_ai_embedding_completion_cost(caplog): litellm.model_cost = litellm.get_model_cost_map(url="") text = "The quick brown fox jumps over the lazy dog." - input_tokens = litellm.token_counter( - model="vertex_ai/text-embedding-004", text=text - ) + input_tokens = litellm.token_counter(model="vertex_ai/text-embedding-004", text=text) model_info = litellm.get_model_info(model="vertex_ai/text-embedding-004") - print("\nExpected model info:\n{}\n\n".format(model_info)) + print(f"\nExpected model info:\n{model_info}\n\n") expected_input_cost = input_tokens * model_info["input_cost_per_token"] @@ -7358,16 +7349,13 @@ def test_vertex_ai_embedding_completion_cost(caplog): ) assert round(expected_input_cost, 6) == round(calculated_input_cost, 6) - print("expected_input_cost: {}".format(expected_input_cost)) - print("calculated_input_cost: {}".format(calculated_input_cost)) + print(f"expected_input_cost: {expected_input_cost}") + print(f"calculated_input_cost: {calculated_input_cost}") captured_logs = [rec.message for rec in caplog.records] for item in captured_logs: - print("\nitem:{}\n".format(item)) - if ( - "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " - in item - ): + print(f"\nitem:{item}\n") + if "litellm.litellm_core_utils.llm_cost_calc.google.cost_per_character(): Exception occured " in item: raise Exception("Error log raised for calculating embedding cost") @@ -7409,9 +7397,7 @@ def test_vertex_ai_mistral_predict_cost(usage): else: from openai.types.completion_usage import CompletionUsage - response_usage = CompletionUsage( - prompt_tokens=32, completion_tokens=55, total_tokens=87 - ) + response_usage = CompletionUsage(prompt_tokens=32, completion_tokens=55, total_tokens=87) response_object = ModelResponse( id="26c0ef045020429d9c5c9b078c01e564", choices=[ @@ -7445,9 +7431,7 @@ def test_vertex_ai_mistral_predict_cost(usage): assert predictive_cost > 0 -@pytest.mark.parametrize( - "model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"] -) +@pytest.mark.parametrize("model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"]) def test_completion_cost_tts(model): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -7546,9 +7530,7 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): cost_1 = completion_cost(model=model, completion_response=response_1) - _model_info = litellm.get_model_info( - model=model, custom_llm_provider=custom_llm_provider - ) + _model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) expected_cost = ( ( response_1.usage.prompt_tokens @@ -7556,12 +7538,9 @@ def test_completion_cost_prompt_caching(model, custom_llm_provider): - response_1.usage.prompt_tokens_details.cache_creation_tokens ) * _model_info["input_cost_per_token"] - + (response_1.usage.prompt_tokens_details.cached_tokens or 0) - * _model_info["cache_read_input_token_cost"] - + (response_1.usage.cache_creation_input_tokens or 0) - * _model_info["cache_creation_input_token_cost"] - + (response_1.usage.completion_tokens or 0) - * _model_info["output_cost_per_token"] + + (response_1.usage.prompt_tokens_details.cached_tokens or 0) * _model_info["cache_read_input_token_cost"] + + (response_1.usage.cache_creation_input_tokens or 0) * _model_info["cache_creation_input_token_cost"] + + (response_1.usage.completion_tokens or 0) * _model_info["output_cost_per_token"] ) # Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) assert round(expected_cost, 5) == round(cost_1, 5) @@ -7653,9 +7632,7 @@ def test_completion_cost_databricks_embedding(model, monkeypatch): sync_handler = HTTPHandler() with patch.object(HTTPHandler, "post", return_value=mock_response): - resp = litellm.embedding( - model=model, input=["hey, how's it going?"], client=sync_handler - ) + resp = litellm.embedding(model=model, input=["hey, how's it going?"], client=sync_handler) print(resp) cost = completion_cost(completion_response=resp) @@ -7745,11 +7722,9 @@ def test_cost_openai_prompt_caching(): usage = response_2.usage _expected_cost2 = ( - (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) - * model_info["input_cost_per_token"] + (usage.prompt_tokens - usage.prompt_tokens_details.cached_tokens) * model_info["input_cost_per_token"] + usage.completion_tokens * model_info["output_cost_per_token"] - + usage.prompt_tokens_details.cached_tokens - * model_info["cache_read_input_token_cost"] + + usage.prompt_tokens_details.cached_tokens * model_info["cache_read_input_token_cost"] ) print("_expected_cost2", _expected_cost2) @@ -7790,9 +7765,7 @@ def test_completion_cost_azure_ai_rerank(model): }, ) print("response", response) - cost = completion_cost( - model=model, completion_response=response, call_type="arerank" - ) + cost = completion_cost(model=model, completion_response=response, call_type="arerank") assert cost > 0 @@ -8744,9 +8717,7 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): completion_tokens=34, prompt_tokens=16, total_tokens=50, - completion_tokens_details=CompletionTokensDetailsWrapper( - audio_tokens=28, reasoning_tokens=0, text_tokens=6 - ), + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=28, reasoning_tokens=0, text_tokens=6), prompt_tokens_details=PromptTokensDetailsWrapper( audio_tokens=0, cached_tokens=0, text_tokens=16, image_tokens=0 ), @@ -8785,27 +8756,15 @@ async def test_test_completion_cost_gpt4o_audio_output_from_model(stream): print(f"model_info: {model_info}") ## input cost - input_audio_cost = ( - model_info["input_cost_per_audio_token"] - * usage_object.prompt_tokens_details.audio_tokens - ) - input_text_cost = ( - model_info["input_cost_per_token"] - * usage_object.prompt_tokens_details.text_tokens - ) + input_audio_cost = model_info["input_cost_per_audio_token"] * usage_object.prompt_tokens_details.audio_tokens + input_text_cost = model_info["input_cost_per_token"] * usage_object.prompt_tokens_details.text_tokens total_input_cost = input_audio_cost + input_text_cost ## output cost - output_audio_cost = ( - model_info["output_cost_per_audio_token"] - * usage_object.completion_tokens_details.audio_tokens - ) - output_text_cost = ( - model_info["output_cost_per_token"] - * usage_object.completion_tokens_details.text_tokens - ) + output_audio_cost = model_info["output_cost_per_audio_token"] * usage_object.completion_tokens_details.audio_tokens + output_text_cost = model_info["output_cost_per_token"] * usage_object.completion_tokens_details.text_tokens total_output_cost = output_audio_cost + output_text_cost @@ -8868,7 +8827,7 @@ def test_completion_cost_model_response_cost(response_model, custom_llm_provider def test_select_model_name_for_cost_calc(): from litellm.cost_calculator import select_model_name_for_cost_calc - from litellm.types.utils import ModelResponse, Choices, Usage, Message + from litellm.types.utils import Choices, Message, ModelResponse, Usage args = { "model": "Mistral-large-nmefg", @@ -8930,9 +8889,7 @@ def test_cost_calculator_with_base_model_with_router(base_model_arg): } if base_model_arg == "litellm_param": - model_item["litellm_params"][ - "base_model" - ] = "bedrock/anthropic.claude-sonnet-5" + model_item["litellm_params"]["base_model"] = "bedrock/anthropic.claude-sonnet-5" elif base_model_arg == "model_info": model_item["model_info"] = { "base_model": "bedrock/anthropic.claude-sonnet-5", @@ -8998,9 +8955,7 @@ def isolated_pricing_registry(local_model_cost_map: None, monkeypatch: pytest.Mo _CUSTOM_PRICING_MODEL: Final = "migration-custom-pricing-chat" -_CUSTOM_PRICING_RATES: Final = MappingProxyType( - {"input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000032} -) +_CUSTOM_PRICING_RATES: Final = MappingProxyType({"input_cost_per_token": 0.0000008, "output_cost_per_token": 0.0000032}) _CUSTOM_PRICING_EXPECTED_COST: Final = 7 * 0.0000008 + 4 * 0.0000032 @@ -9021,7 +8976,13 @@ def _custom_pricing_upstream(request: httpx.Request) -> httpx.Response: body: Final = "".join( ( _custom_pricing_chunk( - [{"index": 0, "delta": {"role": "assistant", "content": "I am ready to help."}, "finish_reason": None}] + [ + { + "index": 0, + "delta": {"role": "assistant", "content": "I am ready to help."}, + "finish_reason": None, + } + ] ), _custom_pricing_chunk([{"index": 0, "delta": {}, "finish_reason": "stop"}]), _custom_pricing_chunk([], {"prompt_tokens": 7, "completion_tokens": 4, "total_tokens": 11}), @@ -9037,7 +8998,11 @@ def _custom_pricing_upstream(request: httpx.Request) -> httpx.Response: "created": 1, "model": _CUSTOM_PRICING_MODEL, "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "I am ready to help."}, "finish_reason": "stop"} + { + "index": 0, + "message": {"role": "assistant", "content": "I am ready to help."}, + "finish_reason": "stop", + } ], "usage": {"prompt_tokens": 7, "completion_tokens": 4, "total_tokens": 11}, }, @@ -9151,7 +9116,12 @@ def test_fireworks_completion_cost_uses_registered_rate(monkeypatch: pytest.Monk monkeypatch.setitem( litellm.model_cost, model, - {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "litellm_provider": "fireworks_ai", "mode": "chat"}, + { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "litellm_provider": "fireworks_ai", + "mode": "chat", + }, ) respx.post("https://api.fireworks.ai/inference/v1/chat/completions").mock( return_value=httpx.Response( @@ -9285,16 +9255,13 @@ def test_speech_cost_with_custom_pricing_bills_the_reported_usage() -> None: ) assert cost == pytest.approx( - usage.prompt_tokens * rates["input_cost_per_token"] - + usage.completion_tokens * rates["output_cost_per_token"] + usage.prompt_tokens * rates["input_cost_per_token"] + usage.completion_tokens * rates["output_cost_per_token"] ) def test_moderation_cost_uses_test_owned_token_rate() -> None: prompt: Final = "moderation input" - rates: Final = MappingProxyType( - {"input_cost_per_token": 0.001, "output_cost_per_token": 0.0} - ) + rates: Final = MappingProxyType({"input_cost_per_token": 0.001, "output_cost_per_token": 0.0}) cost: Final = completion_cost( completion_response=None, @@ -9312,9 +9279,7 @@ def test_moderation_cost_uses_test_owned_token_rate() -> None: def test_vertex_predictive_cost_uses_test_owned_token_rate() -> None: model: Final = "vertex_ai/migration-llama" messages: Final = [{"role": "user", "content": "price control"}] - rates: Final = MappingProxyType( - {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} - ) + rates: Final = MappingProxyType({"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}) cost: Final = completion_cost( model=model, @@ -9322,9 +9287,7 @@ def test_vertex_predictive_cost_uses_test_owned_token_rate() -> None: custom_llm_provider="vertex_ai", custom_cost_per_token=rates, ) - expected_cost: Final = ( - litellm.token_counter(model=model, messages=messages) * 0.001 - ) + expected_cost: Final = litellm.token_counter(model=model, messages=messages) * 0.001 assert cost == pytest.approx(expected_cost) @@ -9333,9 +9296,7 @@ def test_vertex_predictive_cost_uses_test_owned_token_rate() -> None: @pytest.mark.asyncio @pytest.mark.usefixtures("isolated_pricing_registry") @respx.mock -async def test_custom_pricing_cost_reaches_success_callback( - async_mode: bool, monkeypatch: pytest.MonkeyPatch -) -> None: +async def test_custom_pricing_cost_reaches_success_callback(async_mode: bool, monkeypatch: pytest.MonkeyPatch) -> None: model: Final = "migration-callback-pricing-model" litellm.register_model( { @@ -9377,9 +9338,7 @@ async def test_custom_pricing_cost_reaches_success_callback( "input_cost_per_token": 0.001, "output_cost_per_token": 0.002, } - response: Final = ( - await litellm.acompletion(**request) if async_mode else litellm.completion(**request) - ) + response: Final = await litellm.acompletion(**request) if async_mode else litellm.completion(**request) await GLOBAL_LOGGING_WORKER.flush() assert await asyncio.to_thread(callback.success_event.wait, 10.0) @@ -9475,7 +9434,7 @@ def test_json_valid_model_cost_map(): json_str = json.dumps(model_cost) json.loads(json_str) except json.JSONDecodeError as e: - pytest.fail(f"Invalid JSON format: {str(e)}") + pytest.fail(f"Invalid JSON format: {e!s}") def test_batch_cost_calculator(): @@ -9780,9 +9739,7 @@ def test_xai_realtime_bills_text_messages(_xai_voice_test_model: None) -> None: assert cost == pytest.approx(30 * 0.0013333333333333333 + 0.004) -def test_xai_realtime_missing_rates_bills_nothing( - _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_xai_realtime_missing_rates_bills_nothing(_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem( litellm.model_cost, "grok-voice-unpriced", @@ -9877,9 +9834,8 @@ def test_live_session_deduplicates_delegated_responses_and_charges_voice(_local_ ) live_rate: Final = cast(float, litellm.model_cost["gpt-live-1"]["input_cost_per_second"]) backend_model: Final = litellm.model_cost["gpt-4o-mini"] - expected_backend_cost: Final = ( - 10 * cast(float, backend_model["input_cost_per_token"]) - + 5 * cast(float, backend_model["output_cost_per_token"]) + expected_backend_cost: Final = 10 * cast(float, backend_model["input_cost_per_token"]) + 5 * cast( + float, backend_model["output_cost_per_token"] ) cost: Final = handle_realtime_stream_cost_calculation( @@ -10559,12 +10515,18 @@ def _reported_cost_stream_chunk(content: str | None, finish_reason: str | None, model="reported-cost-fixture", created=1, object="chat.completion.chunk", - choices=[litellm.types.utils.StreamingChoices(index=0, delta=litellm.types.utils.Delta(content=content), finish_reason=finish_reason)], + choices=[ + litellm.types.utils.StreamingChoices( + index=0, delta=litellm.types.utils.Delta(content=content), finish_reason=finish_reason + ) + ], usage=usage, ) -def _bill_reported_cost(stream: bool, litellm_params: Mapping[str, object], hidden_params: Mapping[str, object]) -> float | None: +def _bill_reported_cost( + stream: bool, litellm_params: Mapping[str, object], hidden_params: Mapping[str, object] +) -> float | None: """The spend the proxy logs for one upstream reply carrying usage.cost, streamed or not.""" logging_obj: Final = _reported_cost_logging(stream, litellm_params) if stream: @@ -10604,7 +10566,9 @@ def test_provider_reported_usage_cost_is_billed_on_stream_and_non_stream( @pytest.mark.parametrize("stream", [False, True]) -def test_deployment_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: +def test_deployment_pricing_beats_the_provider_reported_usage_cost( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: """Streams used to bill usage.cost even when the operator priced the deployment; both paths bill the operator's rates.""" deployment_id: Final = "reported-cost-deployment-priced-by-the-operator" monkeypatch.setitem( @@ -10624,7 +10588,9 @@ def test_deployment_pricing_beats_the_provider_reported_usage_cost(monkeypatch: @pytest.mark.parametrize("stream", [False, True]) -def test_base_model_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: +def test_base_model_pricing_beats_the_provider_reported_usage_cost( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: base_model: Final = "openai/reported-cost-base-model-fixture" monkeypatch.setitem( litellm.model_cost, @@ -10643,34 +10609,21 @@ def test_provider_reported_usage_cost_takes_discount_then_margin(monkeypatch: py assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037 * 0.8 * 1.5 + 0.001) -def test_a_usage_cost_litellm_stamped_on_a_stream_is_not_billed_again_as_a_provider_report( +def test_the_stream_builder_bills_the_operator_rates_and_leaves_usage_cost_as_the_upstream_reported_it( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) - monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.002}}) - logging_obj: Final = _reported_cost_logging(True, {}) - token_math_plus_margin: Final = 100 * 1e-6 + 20 * 3e-6 + 0.002 - - response: Final[object] = litellm.stream_chunk_builder( - [ - _reported_cost_stream_chunk("hi", None, None), - _reported_cost_stream_chunk(None, "stop", None), - _reported_cost_stream_chunk(None, None, Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)), - ], - messages=[{"role": "user", "content": "hi"}], - logging_obj=logging_obj, + """The bill rides in hidden response_cost. usage.cost stays the upstream's figure, so pricing the assembled + stream again (an MCP round, a logging copy) reads a provider report, never litellm's own bill.""" + deployment_id: Final = "reported-cost-deployment-priced-by-the-operator" + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, ) - - assert isinstance(response, ModelResponse) - assert response.usage is not None and response.usage.cost == pytest.approx(token_math_plus_margin) - assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx( - token_math_plus_margin + logging_obj: Final = _reported_cost_logging( + True, {"input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6, "model_info": {"id": deployment_id}} ) - - -def test_a_zero_bill_is_stamped_over_the_provider_reported_usage_cost_on_a_stream(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 1.0}) - logging_obj: Final = _reported_cost_logging(True, {}) + operator_bill: Final = 100 * 2e-6 + 20 * 6e-6 response: Final[object] = litellm.stream_chunk_builder( [ @@ -10683,195 +10636,11 @@ def test_a_zero_bill_is_stamped_over_the_provider_reported_usage_cost_on_a_strea ) assert isinstance(response, ModelResponse) - assert response.usage is not None and response.usage.cost == 0.0 - assert logging_obj.response_cost_calculator(result=response) == 0.0 - - -_PRICED_DEPLOYMENT_ID: Final = "reported-cost-deployment-priced-by-the-operator" -_PRICED_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType( - { - "input_cost_per_token": 2e-6, - "output_cost_per_token": 6e-6, - "metadata": {"model_info": {"id": _PRICED_DEPLOYMENT_ID}}, - } -) - - -def _reported_cost_usage() -> Usage: - return Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037) - - -def _priced_logging(stream: bool, call_type: str) -> Logging: - """The logging object the proxy hands a priced deployment's call, the deployment under metadata.model_info.""" - logging_obj: Final = _reported_cost_logging(stream, _PRICED_LITELLM_PARAMS) - logging_obj.call_type = call_type - return logging_obj - - -def _reported_cost_responses_response() -> ResponsesAPIResponse: - return ResponsesAPIResponse( - id="resp_reported", - created_at=1, - status="completed", - model="reported-cost-fixture", - object="response", - output=[], - usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=0.037), + response._hidden_params.update({"custom_llm_provider": "openai", "model_id": deployment_id}) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx( + operator_bill ) - - -def _client_visible_reply(route: str) -> tuple[object, Logging]: - """The object a client of the given route reads usage.cost from, after litellm priced it.""" - match route: - case "chat": - logging_obj: Logging = _priced_logging(False, "completion") - response: object = ModelResponse( - id="chatcmpl-reported", - model="reported-cost-fixture", - choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], - usage=_reported_cost_usage(), - ) - update_response_metadata( - result=response, - logging_obj=logging_obj, - model=_REPORTED_COST_FIXTURE_MODEL, - kwargs={}, - start_time=datetime.datetime(2026, 1, 1), - end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), - ) - case "chat-stream": - logging_obj = _priced_logging(True, "completion") - response = litellm.stream_chunk_builder( - [ - _reported_cost_stream_chunk("hi", None, None), - _reported_cost_stream_chunk(None, "stop", None), - _reported_cost_stream_chunk(None, None, _reported_cost_usage()), - ], - messages=[{"role": "user", "content": "hi"}], - logging_obj=logging_obj, - ) - case "responses": - logging_obj = _priced_logging(False, "aresponses") - response = _reported_cost_responses_response() - update_response_metadata( - result=response, - logging_obj=logging_obj, - model=_REPORTED_COST_FIXTURE_MODEL, - kwargs={}, - start_time=datetime.datetime(2026, 1, 1), - end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), - ) - case _: - logging_obj = _priced_logging(True, "aresponses") - response = _reported_cost_responses_response() - stamp_responses_usage_cost(response, logging_obj) - return response, logging_obj - - -@pytest.mark.parametrize("route", ["chat", "chat-stream", "responses", "responses-stream"]) -def test_the_client_reads_what_litellm_billed_from_usage_cost_on_every_route( - monkeypatch: pytest.MonkeyPatch, route: str -) -> None: - """A priced deployment showed its bill on chat streams only; the other three routes still showed the upstream's number.""" - monkeypatch.setitem( - litellm.model_cost, - _PRICED_DEPLOYMENT_ID, - {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, - ) - bill: Final = 100 * 2e-6 + 20 * 6e-6 - - response, logging_obj = _client_visible_reply(route) - - usage: Final = getattr(response, "usage", None) - assert usage is not None and usage.cost == pytest.approx(bill) - assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) - - -def test_a_plain_reply_without_an_upstream_usage_cost_keeps_none_after_pricing(monkeypatch: pytest.MonkeyPatch) -> None: - """Clients comparing the usage dict to the upstream's byte for byte must not find a cost litellm added.""" - monkeypatch.setitem( - litellm.model_cost, - _PRICED_DEPLOYMENT_ID, - {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, - ) - response: Final = ModelResponse( - id="chatcmpl-unreported", - model="reported-cost-fixture", - choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], - usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120), - ) - - update_response_metadata( - result=response, - logging_obj=_priced_logging(False, "completion"), - model=_REPORTED_COST_FIXTURE_MODEL, - kwargs={}, - start_time=datetime.datetime(2026, 1, 1), - end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), - ) - - assert response.usage is not None and getattr(response.usage, "cost", None) is None - assert response._hidden_params["response_cost"] == pytest.approx(100 * 2e-6 + 20 * 6e-6) - - -def test_include_cost_in_streaming_usage_puts_the_bill_on_a_plain_reply_without_an_upstream_usage_cost( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The switch that injects the bill into streamed usage gives plain replies the same key.""" - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) - monkeypatch.setitem( - litellm.model_cost, - _PRICED_DEPLOYMENT_ID, - {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, - ) - response: Final = ModelResponse( - id="chatcmpl-unreported", - model="reported-cost-fixture", - choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], - usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120), - ) - logging_obj: Final = _priced_logging(False, "completion") - - update_response_metadata( - result=response, - logging_obj=logging_obj, - model=_REPORTED_COST_FIXTURE_MODEL, - kwargs={}, - start_time=datetime.datetime(2026, 1, 1), - end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), - ) - - bill: Final = 100 * 2e-6 + 20 * 6e-6 - assert response.usage is not None and response.usage.cost == pytest.approx(bill) - assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(bill) - - -def test_a_bill_stamped_on_a_plain_reply_survives_the_metadata_pass_for_a_second_pricing( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The metadata pass installs a fresh hidden-params dict on a reply that arrived with none, which dropped the - provider's figure the stamp keeps and let a second pricing pass apply the margin twice.""" - monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"percentage": 0.5}}) - monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False) - logging_obj: Final = _reported_cost_logging(False, {}) - response: Final = ModelResponse( - id="chatcmpl-reported", - model="reported-cost-fixture", - choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], - usage=_reported_cost_usage(), - ) - - update_response_metadata( - result=response, - logging_obj=logging_obj, - model=_REPORTED_COST_FIXTURE_MODEL, - kwargs={}, - start_time=datetime.datetime(2026, 1, 1), - end_time=datetime.datetime(2026, 1, 1, 0, 0, 1), - ) - - assert response.usage is not None and response.usage.cost == pytest.approx(0.037 * 1.5) - assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(0.037 * 1.5) + assert response.usage is not None and response.usage.cost == _REPORTED_USAGE["cost"] def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( From b2e1394b0ee23a83d152451f0af6d069c89ed67e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:42:50 +0000 Subject: [PATCH 14/26] fix(cost): type completion_cost region_name so the typed cost_per_token call passes the type gate --- litellm/cost_calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index cf28dbcd17c..fc305c84ac5 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1609,7 +1609,7 @@ def completion_cost( call_type: CallTypesLiteral | None = None, ### REGION ### custom_llm_provider: str | None = None, - region_name=None, # used for bedrock pricing + region_name: str | None = None, # used for bedrock pricing ### IMAGE GEN ### size: str | None = None, quality: str | None = None, From 5f6afa538fa20f0b9fba1e824daab8d719edf9fb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:13:02 +0000 Subject: [PATCH 15/26] test(cost): the stream builder bills into hidden response_cost and leaves usage.cost to the upstream --- .../azure_ai/test_azure_ai_cost_calculator.py | 10 +- tests/unit/test_main.py | 351 +++++++----------- 2 files changed, 149 insertions(+), 212 deletions(-) diff --git a/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py b/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py index 4f8ffa25abb..cef64f6284f 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_cost_calculator.py @@ -14,7 +14,6 @@ import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.azure_ai.cost_calculator import ( - calculate_azure_model_router_flat_cost, cost_per_token, is_azure_model_router, ) @@ -263,7 +262,14 @@ class TestAzureModelRouterCostBreakdown: assert route.call_count == 1 assert json.loads(route.calls.last.request.content)["model"] == "model-router" assert (final_usage.prompt_tokens, final_usage.completion_tokens) == (5000, 2000) - assert final_usage.cost == pytest.approx(routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9) + assert getattr(final_usage, "cost", None) is None, "the bill reaches a client through the proxy egress only" + assembled: Final = litellm.stream_chunk_builder( + chunks=list(chunks), messages=messages, logging_obj=stream.logging_obj + ) + assert assembled is not None + assert assembled._hidden_params["response_cost"] == pytest.approx( + routed_prompt_cost + routed_completion_cost + ROUTED_FEE, rel=1e-9 + ) @pytest.mark.respx(assert_all_called=True) def test_model_router_completion_reports_routed_cost_plus_flat_fee(self, respx_mock: respx.MockRouter) -> None: diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 1faf6dc12ad..13663cf4dd5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -20,7 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from openai import APITimeoutError, AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, OpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion from pydantic import TypeAdapter @@ -290,9 +290,7 @@ def test_completion_missing_role(openai_api_response): print(f"openai_api_response: {openai_api_response}") - with patch.object( - client, "post", MagicMock(return_value=mock_raw_response) - ) as mock_create: + with patch.object(client, "post", MagicMock(return_value=mock_raw_response)) as mock_create: litellm.completion( model="gpt-4o-mini", messages=[ @@ -446,7 +444,7 @@ async def test_url_with_format_param(model, sync_mode, monkeypatch): else: response = await acompletion(**args, client=client) print(response) - except Exception as e: + except Exception: pass mock_client.assert_called() @@ -2535,9 +2533,7 @@ def test_parallel_function_call_stream_reassembles_each_arguments_payload(): created=1751934860, model="gpt-4.1-mini", object="chat.completion.chunk", - choices=[ - StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason) - ], + choices=[StreamingChoices(index=0, delta=Delta(tool_calls=tool_calls), finish_reason=finish_reason)], ) chunks: Final = [ @@ -2668,9 +2664,7 @@ def test_parallel_function_call_stream_over_responses_bridge_ignores_events_it_d }, ) route: Final = respx.post("https://api.openai.com/v1/responses").mock( - return_value=httpx.Response( - 200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events) - ) + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)) ) response: Final = litellm.completion( @@ -3284,9 +3278,7 @@ async def test_retrying() -> None: ) -def test_anthropic_disable_url_suffix_env_var( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter -) -> None: +def test_anthropic_disable_url_suffix_env_var(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> None: monkeypatch.setenv("ANTHROPIC_API_BASE", "https://api.example.com") default_route: Final = respx_mock.post("https://api.example.com/v1/messages").mock( return_value=httpx.Response( @@ -3549,9 +3541,7 @@ async def test_async_mock_streaming_request(): print(chunk) complete_response += chunk["choices"][0]["delta"]["content"] or "" - assert ( - complete_response == "LiteLLM is awesome" - ), f"Unexpected response got {complete_response}" + assert complete_response == "LiteLLM is awesome", f"Unexpected response got {complete_response}" def test_mock_request_n_greater_than_1(): @@ -4077,7 +4067,10 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): assert request_body["prompt_cache_key"] == "test-cache-key" -_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}} +_TOOL: Final[Mapping[str, object]] = { + "type": "function", + "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}, +} # Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body. _OPENAI_CLIENT_SETTINGS: Final = frozenset( @@ -4137,7 +4130,9 @@ _OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] def test_every_openai_body_param_has_a_forwarding_case(): - body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + body_params: Final = ( + frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS + ) assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, ( "a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving " "completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key " @@ -4896,7 +4891,7 @@ def _stream_builder_logging_obj(model: str = "gpt-4o", custom_llm_provider: str return logging_obj -def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch: pytest.MonkeyPatch): +def test_stream_chunk_builder_bills_a_stream_without_writing_into_its_usage(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) chunks: Final = [ _stream_builder_text_chunk("gpt-4o", "Hello "), @@ -4908,10 +4903,8 @@ def test_stream_chunk_builder_stamps_streaming_usage_cost_by_default(monkeypatch ) assert response is not None - usage_cost: Final = getattr(response.usage, "cost", None) - assert usage_cost is not None - assert usage_cost > 0 - assert response._hidden_params["response_cost"] == pytest.approx(usage_cost) + assert getattr(response.usage, "cost", None) is None, "usage.cost is the upstream's field, the bill rides hidden" + assert response._hidden_params["response_cost"] > 0 def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost(): @@ -4942,7 +4935,8 @@ def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost() ) assert response is not None - usage_cost: Final = getattr(response.usage, "cost", None) + assert getattr(response.usage, "cost", None) is None + usage_cost: Final = response._hidden_params.get("response_cost") assert usage_cost is not None assert usage_cost > 0 @@ -4953,7 +4947,6 @@ def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost() assert breakdown["output_cost"] > 0 assert breakdown["input_cost"] + breakdown["output_cost"] == pytest.approx(usage_cost) assert spend_row_cost == pytest.approx(usage_cost) - assert response._hidden_params.get("response_cost") == pytest.approx(usage_cost) def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable(): @@ -5019,7 +5012,7 @@ def test_stream_chunk_builder_prices_alias_response_at_its_deployment_model(): entry: Final = litellm.model_cost[deployment_model] expected_cost: Final = 40 * entry["input_cost_per_token"] + 5 * entry["output_cost_per_token"] assert response is not None - assert getattr(response.usage, "cost", None) == pytest.approx(expected_cost) + assert getattr(response.usage, "cost", None) is None assert response._hidden_params.get("response_cost") == pytest.approx(expected_cost) @@ -5062,7 +5055,9 @@ def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): assert response._hidden_params["response_cost"] == pytest.approx(0.000704) -def test_stream_chunk_builder_stamps_the_margin_priced_bill_over_the_xai_reported_cost(monkeypatch: pytest.MonkeyPatch): +def test_stream_chunk_builder_bills_the_margin_over_the_xai_reported_cost_it_leaves_in_usage( + monkeypatch: pytest.MonkeyPatch, +): monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5}) usage_chunk: Final = _stream_builder_text_chunk("grok-4", "") usage_chunk.usage = Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7, cost=0.42) @@ -5078,7 +5073,7 @@ def test_stream_chunk_builder_stamps_the_margin_priced_bill_over_the_xai_reporte ) assert response is not None - assert getattr(response.usage, "cost", None) == pytest.approx(0.63) + assert getattr(response.usage, "cost", None) == pytest.approx(0.42), "the upstream's figure stays in usage.cost" assert response._hidden_params.get("response_cost") == pytest.approx(0.63) assert logging_obj.response_cost_calculator(result=response) == pytest.approx(0.63) assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx(0.63) @@ -5584,17 +5579,11 @@ def test_drop_params_false_still_rejects_an_invalid_stream_chunk_size() -> None: def test_acompletion_params(): import inspect - from litellm.types.completion import CompletionRequest - acompletion_params_odict = inspect.signature(acompletion).parameters completion_params_dict = inspect.signature(completion).parameters - acompletion_params = { - name: param.annotation for name, param in acompletion_params_odict.items() - } - completion_params = { - name: param.annotation for name, param in completion_params_dict.items() - } + acompletion_params = {name: param.annotation for name, param in acompletion_params_odict.items()} + completion_params = {name: param.annotation for name, param in completion_params_dict.items()} keys_acompletion = set(acompletion_params.keys()) keys_completion = set(completion_params.keys()) @@ -5648,9 +5637,7 @@ def test_null_role_response(): import openai openai_client = openai.OpenAI() - with patch.object( - openai_client, "post", side_effect=_openai_mock_response - ) as mock_response: + with patch.object(openai_client, "post", side_effect=_openai_mock_response) as mock_response: response = litellm.completion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey! how's it going?"}], @@ -5748,7 +5735,8 @@ def test_completion_perplexity_api(): openai_client = OpenAI() with patch.object( - openai_client, "post", + openai_client, + "post", side_effect=_return_pydantic_obj, ) as mock_client: # litellm.set_verbose= True @@ -5791,12 +5779,10 @@ async def test_openai_compatible_custom_api_base(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", new=MagicMock() - ) as mock_call: + with patch.object(openai_client, "post", new=MagicMock()) as mock_call: try: completion( - model="{provider}/my-vllm-model".format(provider=provider), + model=f"{provider}/my-vllm-model", messages=messages, response_format={"type": "json_object"}, client=openai_client, @@ -5808,7 +5794,7 @@ async def test_openai_compatible_custom_api_base(provider): mock_call.assert_called_once() - print("Call KWARGS - {}".format(mock_call.call_args.kwargs)) + print(f"Call KWARGS - {mock_call.call_args.kwargs}") assert "hello" in mock_call.call_args.kwargs["options"]["extra_json"] @@ -5842,12 +5828,10 @@ async def test_openai_compatible_custom_api_video(provider): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", new=MagicMock() - ) as mock_call: + with patch.object(openai_client, "post", new=MagicMock()) as mock_call: try: completion( - model="{provider}/my-vllm-model".format(provider=provider), + model=f"{provider}/my-vllm-model", messages=messages, response_format={"type": "json_object"}, client=openai_client, @@ -6037,10 +6021,7 @@ async def test_completion_functions_param(): mock_client.assert_called() print(f"mock_client.call_args.kwargs: {mock_client.call_args.kwargs}") assert "tools" in mock_client.call_args.kwargs["json"] - assert ( - "litellm_param_is_function_call" - not in mock_client.call_args.kwargs["json"] - ) + assert "litellm_param_is_function_call" not in mock_client.call_args.kwargs["json"] assert response.choices[0].message.function_call is not None except Exception as e: pytest.fail(f"Error occurred: {e}") @@ -6103,7 +6084,7 @@ def test_bedrock_deepseek_custom_prompt_dict(): messages=messages, client=client, ) - except Exception as e: + except Exception: pass mock_post.assert_called_once() @@ -6116,9 +6097,7 @@ def test_bedrock_deepseek_custom_prompt_dict(): def test_bedrock_deepseek_known_tokenizer_config(monkeypatch): - model = ( - "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" - ) + model = "deepseek_r1/arn:aws:bedrock:us-west-2:888602223428:imported-model/bnnr6463ejgf" from unittest.mock import Mock import httpx @@ -6189,9 +6168,7 @@ def test_completion_anthropic_hanging(): {"role": "function", "name": "get_capital", "content": "Kokoko"}, ] - converted_messages = anthropic_messages_pt( - messages, model="claude-3-sonnet-20240229", llm_provider="anthropic" - ) + converted_messages = anthropic_messages_pt(messages, model="claude-3-sonnet-20240229", llm_provider="anthropic") assert len(converted_messages) == 3 for i, msg in enumerate(converted_messages): @@ -6498,9 +6475,7 @@ def test_completion_logprobs_parses_top_logprobs(): "top_logprobs": 3, "reasoning_effort": "none", } - assert [ - item.token for item in response.choices[0].logprobs.content[0].top_logprobs - ] == ["hello", "hi", "hey"] + assert [item.token for item in response.choices[0].logprobs.content[0].top_logprobs] == ["hello", "hi", "hey"] choice: Final = response.choices[0] assert "logprobs" in choice assert "content" in choice.logprobs @@ -6572,9 +6547,11 @@ def test_completion_logprobs_stream_parses_top_logprobs(): "stream_options": {"include_usage": True}, "stream": True, } - assert [ - item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs - ] == ["hello", "hi", "hey"] + assert [item.token for item in response_chunks[0].choices[0].logprobs.content[0].top_logprobs] == [ + "hello", + "hi", + "hey", + ] @respx.mock @@ -6645,9 +6622,7 @@ async def test_bedrock_converse_parses_scripted_response( "api_key": "test-bedrock-token", "aws_region_name": "us-west-2", } - response: Final = ( - await litellm.acompletion(**request) if async_mode else litellm.completion(**request) - ) + response: Final = await litellm.acompletion(**request) if async_mode else litellm.completion(**request) assert route.calls[0].request.headers["authorization"] == "Bearer test-bedrock-token" assert json.loads(route.calls[0].request.content) == { @@ -6734,9 +6709,7 @@ def test_bedrock_converse_parses_scripted_tool_call_arguments( tool_call: Final = response.choices[0].message.tool_calls[0] assert route.call_count == 1 assert tool_call.function.name == "trade" - assert json.loads(tool_call.function.arguments) == { - "orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}] - } + assert json.loads(tool_call.function.arguments) == {"orders": [{"action": "buy", "asset": "BTC", "amount": 0.1}]} @pytest.mark.parametrize( @@ -6774,9 +6747,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( expected_request_body: dict[str, object], fake_provider_credentials: None, ): - route: Final = respx.post( - f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke" - ).mock(return_value=httpx.Response(200, json=response_body)) + route: Final = respx.post(f"https://bedrock-runtime.us-west-2.amazonaws.com/model/{invoke_model_id}/invoke").mock( + return_value=httpx.Response(200, json=response_body) + ) response: Final = litellm.completion( model=model, @@ -6787,9 +6760,7 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( ) request: Final = route.calls[0].request - assert request.headers["authorization"].startswith( - "AWS4-HMAC-SHA256 Credential=unit-test/" - ) + assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 Credential=unit-test/") assert "/us-west-2/bedrock/aws4_request" in request.headers["authorization"] assert request.headers["x-amz-date"].endswith("Z") assert json.loads(request.content) == expected_request_body @@ -6799,9 +6770,9 @@ def test_bedrock_invoke_signs_request_and_parses_scripted_response( @respx.mock def test_completion_azure_ad_token_is_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) litellm.completion( model="azure/gpt-6.1-sol", @@ -6816,9 +6787,9 @@ def test_completion_azure_ad_token_is_forwarded(openai_api_response): @respx.mock def test_completion_azure_extra_headers_are_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) litellm.completion( model="azure/gpt-6.1-sol", @@ -6839,9 +6810,9 @@ def test_completion_azure_extra_headers_are_forwarded(openai_api_response): @respx.mock def test_completion_azure_api_key_argument_is_forwarded(openai_api_response): - route: Final = respx.post( - "https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions" - ).mock(return_value=httpx.Response(200, json=openai_api_response)) + route: Final = respx.post("https://example.openai.azure.com/openai/deployments/gpt-6.1-sol/chat/completions").mock( + return_value=httpx.Response(200, json=openai_api_response) + ) response: Final = litellm.completion( model="azure/gpt-6.1-sol", @@ -7498,9 +7469,7 @@ def test_completion_deep_infra(drop_params): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock( - return_value=httpx.Response(200, json=response_body) - ) + route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) if not drop_params: with pytest.raises(litellm.exceptions.UnsupportedParamsError): @@ -7564,9 +7533,7 @@ def test_completion_deep_infra_mistral(): ], "usage": {"prompt_tokens": 20, "completion_tokens": 5, "total_tokens": 25}, } - route: Final = respx.post(endpoint).mock( - return_value=httpx.Response(200, json=response_body) - ) + route: Final = respx.post(endpoint).mock(return_value=httpx.Response(200, json=response_body)) response: Final = completion( model=model_name, @@ -7615,19 +7582,13 @@ def test_unified_auth_params(provider, model, project, region_name, token): translated_optional_params = litellm.utils.get_optional_params(**data) if provider == "azure": - special_auth_params = ( - litellm.AzureOpenAIConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.AzureOpenAIConfig().get_mapped_special_auth_params() elif provider == "bedrock": - special_auth_params = ( - litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.AmazonBedrockGlobalConfig().get_mapped_special_auth_params() elif provider == "vertex_ai": special_auth_params = litellm.VertexAIConfig().get_mapped_special_auth_params() elif provider == "watsonx": - special_auth_params = ( - litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() - ) + special_auth_params = litellm.IBMWatsonXAIConfig().get_mapped_special_auth_params() for param, value in special_auth_params.items(): assert param in data @@ -7659,9 +7620,7 @@ async def test_dynamic_azure_params(stream, sync_mode): mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF CLIENT IS USED (NO PARAM CHANGE) - with patch.object( - client.chat.completions.with_raw_response, "create", new=mock_client - ) as mock_client: + with patch.object(client.chat.completions.with_raw_response, "create", new=mock_client) as mock_client: try: # client.chat.completions.with_raw_response.create = mock_client if sync_mode: @@ -7690,9 +7649,7 @@ async def test_dynamic_azure_params(stream, sync_mode): new_mock_client = AsyncMock(return_value="Hello world!") ## CHECK IF NEW CLIENT IS USED (PARAM CHANGE) - with patch.object( - client.chat.completions.with_raw_response, "create", new=new_mock_client - ) as new_mock_client: + with patch.object(client.chat.completions.with_raw_response, "create", new=new_mock_client) as new_mock_client: try: if sync_mode: _ = completion( @@ -7903,9 +7860,7 @@ def test_completion_novita_ai(): openai_client = OpenAI(api_key="fake-key") - with patch.object( - openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse) - ) as mock_call: + with patch.object(openai_client, "post", return_value=MagicMock(spec=LegacyAPIResponse)) as mock_call: mock_call.return_value.headers = {} mock_call.return_value.parse.return_value = litellm.ModelResponse( choices=[{"message": {"role": "assistant", "content": "Hello"}}] @@ -7922,10 +7877,7 @@ def test_completion_novita_ai(): assert response.choices[0].message.content == "Hello" # Verify model is passed correctly - assert ( - mock_call.call_args.kwargs["body"]["model"] - == "meta-llama/llama-3.3-70b-instruct" - ) + assert mock_call.call_args.kwargs["body"]["model"] == "meta-llama/llama-3.3-70b-instruct" # Verify messages are passed correctly assert mock_call.call_args.kwargs["body"]["messages"] == messages @@ -7982,9 +7934,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): litellm.set_verbose = True monkeypatch.setattr(litellm, "enable_preview_features", enable_preview_features) - with patch.object( - client, "post", return_value=MagicMock() - ) as mock_completion: + with patch.object(client, "post", return_value=MagicMock()) as mock_completion: try: resp = litellm.completion( model="openai/gpt-3.5-turbo", @@ -7997,9 +7947,7 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): mock_completion.assert_called_once() if enable_preview_features: - assert mock_completion.call_args.kwargs["body"]["metadata"] == { - "my-test-key": "my-test-value" - } + assert mock_completion.call_args.kwargs["body"]["metadata"] == {"my-test-key": "my-test-value"} else: assert "metadata" not in mock_completion.call_args.kwargs["body"] @@ -8220,17 +8168,30 @@ async def test_async_responses_still_retries_provider_server_errors(monkeypatch) monkeypatch.setattr(litellm, "num_retries", None) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) with respx.mock as upstream: - response = upstream.post("https://openai-test.invalid/v1/responses").mock(side_effect=[ - httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), - httpx.Response(200, json={ - "id": "resp-retry", "object": "response", "created_at": 1, "status": "completed", - "model": "test-model", "output": [], - "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, - }), - ]) + response = upstream.post("https://openai-test.invalid/v1/responses").mock( + side_effect=[ + httpx.Response(500, json={"error": {"message": "temporary provider failure", "type": "server_error"}}), + httpx.Response( + 200, + json={ + "id": "resp-retry", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "test-model", + "output": [], + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + }, + ), + ] + ) result = await litellm.aresponses( - model="openai/test-model", input="ping", api_key="test-key", - api_base="https://openai-test.invalid/v1", num_retries=1, max_retries=0, + model="openai/test-model", + input="ping", + api_key="test-key", + api_base="https://openai-test.invalid/v1", + num_retries=1, + max_retries=0, ) assert result.status == "completed" assert response.call_count == 2 @@ -8416,9 +8377,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini", - "choices": [ - {"index": 0, "delta": {}, "finish_reason": "function_call"} - ], + "choices": [{"index": 0, "delta": {}, "finish_reason": "function_call"}], }, ) @@ -8442,9 +8401,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ), ) ) - route: Final = respx.post(f"{api_base}/chat/completions").mock( - side_effect=lambda _: next(scripted_responses) - ) + route: Final = respx.post(f"{api_base}/chat/completions").mock(side_effect=lambda _: next(scripted_responses)) stream: Final = litellm.completion( model="gpt-4o-mini", api_key="sk-test", @@ -8455,11 +8412,14 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( ) answer_chunks: Final = tuple(stream) - assert "".join( - chunk.choices[0].delta.content - for chunk in answer_chunks - if chunk.choices and chunk.choices[0].delta.content - ) == "The answer is 42." + assert ( + "".join( + chunk.choices[0].delta.content + for chunk in answer_chunks + if chunk.choices and chunk.choices[0].delta.content + ) + == "The answer is 42." + ) assert tuple( chunk.choices[0].finish_reason for chunk in answer_chunks @@ -8476,12 +8436,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( assert answer_chunks[-1].usage.completion_tokens == 2 assert answer_chunks[-1].usage.total_tokens == 6 assert stream._hidden_params["api_base"] == api_base - assert ( - answer_chunks[0]._hidden_params["additional_headers"][ - "llm_provider-x-ratelimit-remaining-requests" - ] - == "17" - ) + assert answer_chunks[0]._hidden_params["additional_headers"]["llm_provider-x-ratelimit-remaining-requests"] == "17" function_stream: Final = litellm.completion( model="gpt-4o-mini", @@ -8493,9 +8448,7 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( stream=True, ) function_chunks: Final = tuple(function_stream) - assembled: Final = litellm.stream_chunk_builder( - chunks=function_chunks, messages=messages - ) + assembled: Final = litellm.stream_chunk_builder(chunks=function_chunks, messages=messages) function_call: Final = assembled.choices[0].message.function_call assert function_call is not None @@ -8507,12 +8460,8 @@ def test_openai_chat_stream_routes_legacy_function_calls_and_usage( if chunk.choices and chunk.choices[0].finish_reason is not None ) == ("function_call",) assert route.call_count == 2 - assert json.loads(route.calls[0].request.content)["stream_options"] == { - "include_usage": True - } - assert json.loads(route.calls[1].request.content)["function_call"] == { - "name": "get_current_weather" - } + assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} + assert json.loads(route.calls[1].request.content)["function_call"] == {"name": "get_current_weather"} assert json.loads(route.calls[1].request.content)["functions"] == list(functions) @@ -8551,15 +8500,9 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, }, ) - body: Final = "".join( - f"data: {json.dumps(event)}\n\n" for event in events - ) + "data: [DONE]\n\n" - route: Final = respx.post( - "https://api.openai.com/v1/chat/completions" - ).mock( - return_value=httpx.Response( - 200, text=body, headers={"content-type": "text/event-stream"} - ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) ) stream: Final = await litellm.acompletion( model="gpt-4o-mini", @@ -8570,11 +8513,10 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( ) chunks: Final = tuple([chunk async for chunk in stream]) - assert "".join( - chunk.choices[0].delta.content - for chunk in chunks - if chunk.choices and chunk.choices[0].delta.content - ) == "async reply" + assert ( + "".join(chunk.choices[0].delta.content for chunk in chunks if chunk.choices and chunk.choices[0].delta.content) + == "async reply" + ) assert tuple(getattr(chunk, "usage", None) for chunk in chunks[:-1]) == ( None, None, @@ -8582,9 +8524,7 @@ async def test_async_openai_stream_options_include_usage_on_the_last_chunk( assert chunks[-1].usage.prompt_tokens == 3 assert chunks[-1].usage.completion_tokens == 2 assert chunks[-1].usage.total_tokens == 5 - assert json.loads(route.calls[0].request.content)["stream_options"] == { - "include_usage": True - } + assert json.loads(route.calls[0].request.content)["stream_options"] == {"include_usage": True} @respx.mock @@ -8621,16 +8561,10 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], }, ) - body: Final = "".join( - f"data: {json.dumps(event)}\n\n" for event in events - ) + "data: [DONE]\n\n" - return httpx.Response( - 200, text=body, headers={"content-type": "text/event-stream"} - ) + body: Final = "".join(f"data: {json.dumps(event)}\n\n" for event in events) + "data: [DONE]\n\n" + return httpx.Response(200, text=body, headers={"content-type": "text/event-stream"}) - route: Final = respx.post( - "https://api.openai.com/v1/chat/completions" - ).mock(side_effect=respond) + route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(side_effect=respond) async def collect(prompt: str) -> tuple[str, tuple[str, ...]]: stream = await litellm.acompletion( @@ -8640,9 +8574,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( stream=True, ) chunks: Final = tuple([chunk async for chunk in stream]) - response_text: Final = "".join( - chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices - ) + response_text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) finish_reasons: Final = tuple( chunk.choices[0].finish_reason for chunk in chunks @@ -8650,9 +8582,7 @@ async def test_parallel_openai_streams_keep_prompt_responses_separate( ) return response_text, finish_reasons - responses: Final = tuple( - await asyncio.gather(collect("request one"), collect("request two")) - ) + responses: Final = tuple(await asyncio.gather(collect("request one"), collect("request two"))) assert tuple(response for response, _ in responses) == ( "response one", @@ -8825,7 +8755,7 @@ def test_openai_streaming_tool_call_arguments_are_valid_json( @respx.mock -def test_streaming_openai_usage_includes_test_owned_cost( +def test_streaming_openai_usage_carries_no_cost_the_upstream_did_not_report( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -8846,9 +8776,7 @@ def test_streaming_openai_usage_includes_test_owned_cost( "object": "chat.completion.chunk", "created": 1, "model": model_name, - "choices": [ - {"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None} - ], + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "priced"}, "finish_reason": None}], }, { "id": "chatcmpl-cost", @@ -8883,12 +8811,10 @@ def test_streaming_openai_usage_includes_test_owned_cost( ) stream_chunks: Final = tuple(stream) usage_chunks: Final = tuple( - getattr(chunk, "usage", None) - for chunk in stream_chunks - if getattr(chunk, "usage", None) is not None + getattr(chunk, "usage", None) for chunk in stream_chunks if getattr(chunk, "usage", None) is not None ) assert len(usage_chunks) == 1 - assert usage_chunks[0].cost == pytest.approx(4 * 0.001 + 2 * 0.002) + assert getattr(usage_chunks[0], "cost", None) is None, "the bill reaches a client through the proxy egress only" def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock: respx.MockRouter) -> None: @@ -8901,9 +8827,7 @@ def test_completion_ensure_alternating_roles_sends_continue_messages(respx_mock: "object": "chat.completion", "created": 0, "model": "databricks-meta-llama-3-1-70b-instruct", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Sure"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 40, "completion_tokens": 1, "total_tokens": 41}, }, ) @@ -9113,9 +9037,12 @@ def test_azure_ai_forwards_reasoning_effort_once_the_deployment_declares_levels( assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "adaptive", _KIMI_DEPLOYMENT)["reasoning_effort"] == ( "adaptive" ) - assert _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ - "reasoning_effort" - ] == "none" + assert ( + _chat_body_sent(respx_mock, "azure_ai/my-kimi", "none", {"reasoning_effort_levels": ["low", "high"]})[ + "reasoning_effort" + ] + == "none" + ) def test_a_model_map_declaration_alone_leaves_reasoning_effort_to_drop_params( @@ -9250,7 +9177,9 @@ def test_a_default_effort_beside_a_thinking_budget_sends_no_effort_on_a_declarin thinking={"type": "enabled", "budget_tokens": 2048, **display}, **default_effort, ) - adaptive_alone: Final = body_sent(respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display}) + adaptive_alone: Final = body_sent( + respx_mock, _OPUS_DECLARING_LOW_AND_HIGH, thinking={"type": "adaptive", **display} + ) assert "output_config" not in defaulted assert defaulted == adaptive_alone @@ -9755,7 +9684,9 @@ async def test_aspeech_answers_a_non_string_provider_hint_with_a_bad_request( hint: str, respx_mock: respx.MockRouter ) -> None: with pytest.raises(litellm.BadRequestError) as raised: - await litellm.aspeech(model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123}) + await litellm.aspeech( + model="kokoro", input="hello", voice="alloy", api_key="fake_openai_api_key", **{hint: 123} + ) assert raised.value.status_code == 400 assert f"Invalid {hint}" in raised.value.message @@ -9914,7 +9845,9 @@ async def test_converse_fallback_prices_the_current_attempt( } if asynchronous: with pytest.raises(litellm.BadRequestError): - await litellm.acompletion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) + await litellm.acompletion( + model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs + ) else: with pytest.raises(litellm.BadRequestError): litellm.completion(model="bedrock/converse/amazon.nova-micro-v1:0", model_id=failed_profile, **kwargs) @@ -9938,9 +9871,7 @@ async def test_converse_fallback_prices_the_current_attempt( assert result is not None prices: Final = litellm.get_model_info(profile or model, custom_llm_provider="bedrock") expected: Final = ( - 0.001 + 0.002 - if custom_price - else prices["input_cost_per_token"] + prices["output_cost_per_token"] + 0.001 + 0.002 if custom_price else prices["input_cost_per_token"] + prices["output_cost_per_token"] ) active_logging: Final = response.logging_obj if stream else logging_obj assert active_logging.response_cost_calculator(result) == pytest.approx(expected) From a41700c297875b9b3b9fac04463aade435dd99ce Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 27 Oct 2026 21:25:58 +0000 Subject: [PATCH 16/26] test(cost): integration cases for a reported cost of 0 and an openrouter hop on a priced deployment Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_calculation/cases/openai.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index fb7a90a634a..923ce6136aa 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -12027,6 +12027,54 @@ GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED } ) + +_GATEWAY_ZERO_USAGE: Final = {**_GATEWAY_REPORTED_USAGE, "cost": 0} + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced", + "response": JsonResponse( + content_type="application/json", + body={**GATEWAY_REPORTED_COST.response.body, "usage": _GATEWAY_ZERO_USAGE}, + ), + } +) + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced-stream", + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_ZERO_USAGE}, + done=True, + ), + ), + } +) + +_GATEWAY_OPENROUTER_OPERATOR_PRICED: Final = _GATEWAY_OPERATOR_PRICED.model_copy( + update={"model": "openrouter/vendor/gateway-reported-cost-fixture"} +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = ( + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced-stream", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } + ) +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GATEWAY_REPORTED_COST, GATEWAY_REPORTED_COST_STREAM, @@ -12036,6 +12084,10 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GATEWAY_REPORTED_COST_RESPONSES_STREAM, GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12281,6 +12333,13 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( StreamParityTestCase( plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM + ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, + ), StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM), StreamParityTestCase( plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, From 9c6954c00a3011a23fc194234cc7aad8d09b54c6 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 27 Oct 2026 21:39:26 +0000 Subject: [PATCH 17/26] refactor(cost): cut the PR back to billing a reported usage.cost, no client-visible rewrite and provider readers as on main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 7 + litellm/llms/perplexity/cost_calculator.py | 16 ++ litellm/llms/xai/cost_calculator.py | 18 +- litellm/proxy/common_request_processing.py | 11 +- litellm/proxy/proxy_server.py | 20 +- litellm/proxy/usage_cost_egress.py | 77 ------- .../cost_calculation/assertions.py | 16 -- .../test_perplexity_cost_calculator.py | 28 ++- .../test_xai_responses_transformation.py | 6 +- .../llms/xai/test_xai_chat_transformation.py | 6 +- .../unit/llms/xai/test_xai_cost_calculator.py | 10 +- tests/unit/proxy/test_usage_cost_egress.py | 200 ------------------ 12 files changed, 65 insertions(+), 350 deletions(-) delete mode 100644 litellm/proxy/usage_cost_egress.py delete mode 100644 tests/unit/proxy/test_usage_cost_egress.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index fc305c84ac5..d2fdec8688e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -408,7 +408,14 @@ def _per_second_pricing_cost( return resolved_cost_per_second * seconds, 0.0 +_PROVIDERS_WITH_OWN_REPORTED_COST_READER: Final = frozenset( + {LlmProviders.XAI.value, LlmProviders.PERPLEXITY.value, LlmProviders.OPENCODE_GO.value} +) + + def _billable_reported_cost(usage_object: Usage | None, custom_llm_provider: str | None) -> float | None: + if custom_llm_provider in _PROVIDERS_WITH_OWN_REPORTED_COST_READER: + return None return _billable_header_cost(provider_reported_usage_cost(usage_object, custom_llm_provider)) diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index a6825eacfad..2d762adf958 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -24,6 +24,22 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ + ## USE PRE-CALCULATED COST FROM PERPLEXITY IF AVAILABLE + ## Perplexity returns accurate cost in usage.cost.total_cost including request fees. + ## By the time it reaches here, ResponseAPIUsage.parse_cost has already flattened + ## that dict down to a float, so both shapes must be accepted. + cost_info: Final = getattr(usage, "cost", None) + total_cost: float | None = None + if isinstance(cost_info, dict): + total_cost = cost_info.get("total_cost") + elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool): + total_cost = float(cost_info) + if total_cost is not None: + # Return total cost as completion_cost (prompt_cost=0) since Perplexity + # doesn't break down by input/output in their cost object + return (0.0, float(total_cost)) + + ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it ## GET MODEL INFO model_info: Final = get_model_info(model=model, custom_llm_provider="perplexity") diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index b62ed39c4fa..aec535c2086 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -1,8 +1,8 @@ """ Helper util for handling XAI-specific cost calculation +- Prefers the cost xAI reports on the response over recomputing it locally - Uses the generic cost calculator which already handles tiered pricing correctly - Handles XAI-specific reasoning token billing (billed as part of completion tokens) -- Skips the web search surcharge when xAI reported a total that already covers it """ import math @@ -49,6 +49,19 @@ def _cost_reported_by_xai(usage: "Usage") -> float | None: return float(reported_cost) +def _split_reported_cost(model: str, usage: Usage, reported_cost: float) -> tuple[float, float]: + """xAI reports one total, so attribute it to input and output in the ratio our own token math gives.""" + try: + priced_input, priced_output = _token_math_cost_per_token(model, usage) + except Exception: # noqa: BLE001 # generic_cost_per_token raises bare Exception for unmapped models + return 0.0, reported_cost + priced_total: Final = priced_input + priced_output + if priced_total <= 0: + return 0.0, reported_cost + input_cost: Final = reported_cost * priced_input / priced_total + return input_cost, reported_cost - input_cost + + def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: """ Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens. @@ -61,6 +74,9 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ + reported_cost: Final = _cost_reported_by_xai(usage) + if reported_cost is not None: + return _split_reported_cost(model, usage, reported_cost) return _token_math_cost_per_token(model, usage) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c5843da28d9..3b9c17baa02 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -130,7 +130,6 @@ from litellm.proxy.route_llm_request import ( route_request, ) from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id -from litellm.proxy.usage_cost_egress import with_client_usage_cost from litellm.proxy.utils import ( # noqa: F401 # legacy module exports ProxyLogging, _check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -274,7 +273,6 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD -from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( ModelResponse, ModelResponseStream, @@ -3327,7 +3325,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, ) - return with_client_usage_cost(response, response_cost_for_headers) + return response @staticmethod async def _record_container_owners_from_responses_if_needed( @@ -4712,7 +4710,6 @@ class ProxyBaseLLMRequestProcessing: completion_tokens_details=usage.get("completion_tokens_details"), prompt_tokens_details=usage.get("prompt_tokens_details"), server_tool_use=usage.get("server_tool_use"), - cost=usage.get("cost"), ) @staticmethod @@ -4727,7 +4724,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _completion_cost_or_none( - model_response: ModelResponse | ResponsesAPIResponse, model_name: str, service_tier: str | None + model_response: ModelResponse, model_name: str, service_tier: str | None ) -> float | None: try: return litellm.completion_cost( @@ -4738,7 +4735,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _logging_obj_cost_or_none( - model_response: ModelResponse | ResponsesAPIResponse, litellm_logging_obj: LiteLLMLoggingObj + model_response: ModelResponse, litellm_logging_obj: LiteLLMLoggingObj ) -> float | None: # Pricing a frame stamps cost_breakdown and, on failure, the cost-failure debug key onto # the live logging object. The pass-through handlers never recompute either one, so a @@ -4765,7 +4762,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _streamed_usage_cost( - model_response: ModelResponse | ResponsesAPIResponse, + model_response: ModelResponse, model_name: str, service_tier: str | None, litellm_logging_obj: LiteLLMLoggingObj | None, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5c7388f7861..bc79becf7bb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -922,7 +922,6 @@ from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import ( router as user_banner_endpoints_router, ) -from litellm.proxy.usage_cost_egress import Pricer, with_client_stream_usage_cost from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports PrismaClient, ProxyLogging, @@ -10135,22 +10134,6 @@ def _fast_serialize_simple_model_response_stream( return orjson.dumps(payload) -def _stream_usage_pricer(request_data: Mapping[str, object]) -> Pricer: - """Bills a usage-bearing stream event the way the request's logging object bills the spend row.""" - logging_obj: Final = request_data.get("litellm_logging_obj") - model_name: Final = str(request_data.get("model", "")) - - def price(reply: ModelResponse | ResponsesAPIResponse) -> float | None: - return ProxyBaseLLMRequestProcessing._streamed_usage_cost( # pyright: ignore[reportPrivateUsage] # the proxy's one stream pricer - reply, - model_name, - None, - logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, - ) - - return price - - def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: if isinstance(chunk, ModelResponseStream): serialized_chunk: Final = _fast_serialize_simple_model_response_stream(chunk) @@ -10489,7 +10472,6 @@ async def async_data_generator( needs_per_chunk_hook: Final = proxy_logging_obj.needs_per_chunk_streaming_hook() is_raw_sse_stream: Final = bool(request_data.get("_litellm_raw_sse_stream")) strip_stream_usage: Final = bool(request_data.get("_litellm_strip_stream_usage")) - price_stream_usage: Final = _stream_usage_pricer(request_data) raw_sse_buffer = "" # A stream can start on a deployment with keepalive off and fall back @@ -10576,7 +10558,7 @@ async def async_data_generator( error_state.observe_chunk(chunk) raw_passthrough = False if isinstance(chunk, BaseModel): - chunk = _serialize_streaming_chunk(with_client_stream_usage_cost(chunk, price_stream_usage)) + chunk = _serialize_streaming_chunk(chunk) elif isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="replace") if is_raw_sse_stream: diff --git a/litellm/proxy/usage_cost_egress.py b/litellm/proxy/usage_cost_egress.py deleted file mode 100644 index 820fce85974..00000000000 --- a/litellm/proxy/usage_cost_egress.py +++ /dev/null @@ -1,77 +0,0 @@ -"""What a client reads in ``usage.cost``: litellm's bill, written once on the way out of the proxy, onto a copy. - -Inside the pipeline ``usage.cost`` stays what the upstream reported, so pricing a reply again (an MCP round, a -logging copy, a stream rebuilt from its chunks) never reads litellm's own bill back as a provider report. -""" - -from __future__ import annotations - -import math -from collections.abc import Callable -from typing import Final, TypeVar - -from pydantic import BaseModel - -import litellm -from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse -from litellm.types.utils import ModelResponse, ModelResponseStream, Usage - -Pricer = Callable[[ModelResponse | ResponsesAPIResponse], float | None] -"""Prices a reply the way the request's logging object bills it (deployment rates, discount, margin).""" - -_RESPONSES_USAGE_EVENTS: Final = frozenset({"response.completed", "response.incomplete"}) -_ReplyT = TypeVar("_ReplyT", bound=BaseModel) - - -def billable(cost: object) -> float | None: - if isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0: - return None - return float(cost) - - -def client_usage_cost(upstream_cost: object, bill: object, *, wanted: bool = False) -> float | None: - """The bill, when the upstream put a number in usage.cost or the operator wants it in every usage block - (``litellm.include_cost_in_streaming_usage``). None leaves the usage as the upstream sent it.""" - reported: Final = isinstance(upstream_cost, (int, float)) and not isinstance(upstream_cost, bool) - if not reported and not wanted and not litellm.include_cost_in_streaming_usage: - return None - return billable(bill) - - -def _with_usage_cost(reply: _ReplyT, usage: Usage | ResponseAPIUsage, cost: float) -> _ReplyT: - return reply.model_copy(update={"usage": usage.model_copy(update={"cost": cost})}) - - -def with_client_usage_cost(response: object, bill: object) -> object: - """A plain chat or Responses reply as the client gets it: a copy with the bill in usage.cost, or the reply - itself when its usage stays as the upstream sent it.""" - if not isinstance(response, (ModelResponse, ResponsesAPIResponse)): - return response - usage: Final = getattr(response, "usage", None) - if not isinstance(usage, (Usage, ResponseAPIUsage)): - return response - cost: Final = client_usage_cost(getattr(usage, "cost", None), bill) - if cost is None: - return response - return _with_usage_cost(response, usage, cost) - - -def with_client_stream_usage_cost(chunk: BaseModel, price: Pricer) -> BaseModel: - """A streamed event as the client gets it. Only the usage-bearing events are priced: the chat chunk carrying - ``usage`` and the Responses ``response.completed`` / ``response.incomplete`` events, which carry the bill - whether or not the upstream reported one, as they did before the bill moved out here.""" - if isinstance(chunk, ModelResponseStream): - usage: Final = getattr(chunk, "usage", None) - if not isinstance(usage, Usage): - return chunk - cost: Final = client_usage_cost(getattr(usage, "cost", None), price(ModelResponse(usage=usage))) - return chunk if cost is None else _with_usage_cost(chunk, usage, cost) - if getattr(chunk, "type", None) not in _RESPONSES_USAGE_EVENTS: - return chunk - response: Final = getattr(chunk, "response", None) - if not isinstance(response, ResponsesAPIResponse) or not isinstance(response.usage, ResponseAPIUsage): - return chunk - response_cost: Final = client_usage_cost(getattr(response.usage, "cost", None), price(response), wanted=True) - if response_cost is None: - return chunk - return chunk.model_copy(update={"response": _with_usage_cost(response, response.usage, response_cost)}) diff --git a/tests/integration/cost_calculation/assertions.py b/tests/integration/cost_calculation/assertions.py index f37bb32e3b3..6e2af74d1a2 100644 --- a/tests/integration/cost_calculation/assertions.py +++ b/tests/integration/cost_calculation/assertions.py @@ -55,18 +55,6 @@ def assert_streamed_usage_cost(case_name: str, endpoint: str, response_text: str assert approx_equal(seen, spend), f"{case_name}: streamed usage.cost {seen} != spend {spend}" -def assert_json_usage_cost(case_name: str, endpoint: str, response: httpx.Response, spend: float) -> None: - """A JSON reply's usage.cost is what the row billed, the upstream's own figure included, so a client and the - spend log never disagree on a priced deployment.""" - if endpoint not in STREAMED_COST_ENDPOINTS: - return - usage: Final = JSON_OBJECT.validate_json(response.text).get("usage") - cost: Final = usage.get("cost") if isinstance(usage, dict) else None - if not isinstance(cost, (int, float)) or isinstance(cost, bool): - return - assert approx_equal(float(cost), spend), f"{case_name}: usage.cost {cost} != spend {spend}" - - def assert_breakdown( case_name: str, response_content_type: str, @@ -172,8 +160,6 @@ def assert_exact( assert_total_is_sum_of_components(row, breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, expected.spend) - if response is not None and response_content_type == "application/json": - assert_json_usage_cost(case_name, response.request.url.path, response, expected.spend) def assert_recount( @@ -215,5 +201,3 @@ def assert_recount( assert_total_is_sum_of_components(row, row.breakdown, case_name) if response is not None and response_content_type == "text/event-stream": assert_streamed_usage_cost(case_name, response.request.url.path, response.text, row.spend) - if response is not None and response_content_type == "application/json": - assert_json_usage_cost(case_name, response.request.url.path, response, row.spend) diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index cd25b7a9c3d..83c71479311 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -15,7 +15,6 @@ import pytest # Add the project root to Python path import litellm -from litellm.cost_calculator import cost_per_token as billed_cost_per_token from litellm.llms.perplexity.cost_calculator import ( cost_per_token as perplexity_cost_per_token, ) @@ -110,13 +109,13 @@ class TestPerplexityCostCalculator: "total_cost": 0.008, } - prompt_cost, completion_cost = billed_cost_per_token( - model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage - ) + prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage) - # When Perplexity provides total_cost, that total is the bill - assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10) - assert prompt_cost > 0.0 and completion_cost > 0.0 + # When Perplexity provides total_cost, we use it directly + # prompt_cost should be 0, completion_cost should be total_cost + assert prompt_cost == 0.0 + assert completion_cost == 0.008 + assert prompt_cost + completion_cost == 0.008 def test_uses_perplexity_provided_cost_when_normalized_to_float(self): """ @@ -136,11 +135,10 @@ class TestPerplexityCostCalculator: usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) usage.cost = 0.008 - prompt_cost, completion_cost = billed_cost_per_token( - model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage - ) + prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage) - assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10) + assert prompt_cost == 0.0 + assert completion_cost == 0.008 OFF_PEAK_MODEL = "sonar-off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" @@ -228,9 +226,7 @@ class TestPerplexityCostCalculator: usage = Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200) usage.cost = {"total_cost": 0.00501} - prompt_cost, completion_cost = billed_cost_per_token( - model=self.OFF_PEAK_MODEL, custom_llm_provider="perplexity", usage_object=usage - ) + prompt_cost, completion_cost = perplexity_cost_per_token(model=self.OFF_PEAK_MODEL, usage=usage) - assert math.isclose(prompt_cost + completion_cost, 0.00501, rel_tol=1e-10) - assert prompt_cost > 0.0 and completion_cost > 0.0 + assert prompt_cost == 0.0 + assert completion_cost == 0.00501 diff --git a/tests/unit/llms/xai/responses/test_xai_responses_transformation.py b/tests/unit/llms/xai/responses/test_xai_responses_transformation.py index 95ff98ac024..188ca453364 100644 --- a/tests/unit/llms/xai/responses/test_xai_responses_transformation.py +++ b/tests/unit/llms/xai/responses/test_xai_responses_transformation.py @@ -11,7 +11,7 @@ from unittest.mock import MagicMock, Mock import httpx -from litellm.cost_calculator import cost_per_token +from litellm.llms.xai.cost_calculator import cost_per_token from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( @@ -498,7 +498,7 @@ class TestXAIResponsesReportedCost: assert usage.cost == 0.0037756 chat_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage) - assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=chat_usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) def test_streamed_reported_cost_reaches_the_cost_calculator(self): event = XAIResponsesAPIConfig().transform_streaming_response( @@ -520,7 +520,7 @@ class TestXAIResponsesReportedCost: assert isinstance(event, ResponseCompletedEvent) chat_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(event.response.usage) - assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=chat_usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) def test_usage_without_a_reported_cost_is_left_alone(self): usage = self._transformed_usage({"input_tokens": 100, "output_tokens": 200, "total_tokens": 300}) diff --git a/tests/unit/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py index 3aa35759662..69e620dbc8a 100644 --- a/tests/unit/llms/xai/test_xai_chat_transformation.py +++ b/tests/unit/llms/xai/test_xai_chat_transformation.py @@ -13,7 +13,7 @@ from litellm.llms.xai.chat.transformation import ( XAIChatCompletionStreamingHandler, XAIChatConfig, ) -from litellm.cost_calculator import cost_per_token +from litellm.llms.xai.cost_calculator import cost_per_token from litellm.types.utils import ( CompletionTokensDetailsWrapper, ModelResponse, @@ -433,7 +433,7 @@ class TestXAIReportedCost: ) assert usage.cost == 0.0037756 - assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0037756) def test_usage_without_a_reported_cost_is_left_alone(self): usage = self._transformed_usage({"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}) @@ -481,7 +481,7 @@ class TestXAIReportedCost: assembled = litellm.stream_chunk_builder(chunks=[parsed]) assert assembled.usage.cost == 0.0037756 - assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=assembled.usage) == ( + assert cost_per_token(model="grok-4-latest", usage=assembled.usage) == ( 0.0, 0.0037756, ) diff --git a/tests/unit/llms/xai/test_xai_cost_calculator.py b/tests/unit/llms/xai/test_xai_cost_calculator.py index c1d2ee5f63d..5219dba42ba 100644 --- a/tests/unit/llms/xai/test_xai_cost_calculator.py +++ b/tests/unit/llms/xai/test_xai_cost_calculator.py @@ -6,7 +6,6 @@ import math import os import litellm -from litellm.cost_calculator import cost_per_token as billed_cost_per_token from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -116,9 +115,7 @@ class TestXAICostCalculator: model="grok-4.7", usage=Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) ) - prompt_cost, completion_cost = billed_cost_per_token( - model="xai/grok-4.7", custom_llm_provider="xai", usage_object=usage - ) + prompt_cost, completion_cost = cost_per_token(model="grok-4.7", usage=usage) assert math.isclose(prompt_cost + completion_cost, 0.0037756, rel_tol=1e-10) assert math.isclose(prompt_cost / completion_cost, priced_prompt / priced_completion, rel_tol=1e-10) @@ -168,10 +165,7 @@ class TestXAICostCalculator: """A reported zero is a real answer, not a missing value.""" usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300, cost=0.0) - assert billed_cost_per_token(model="xai/grok-4-latest", custom_llm_provider="xai", usage_object=usage) == ( - 0.0, - 0.0, - ) + assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0) def test_custom_pricing_beats_the_reported_cost(self): response = ModelResponse( diff --git a/tests/unit/proxy/test_usage_cost_egress.py b/tests/unit/proxy/test_usage_cost_egress.py deleted file mode 100644 index 3c62ddb9308..00000000000 --- a/tests/unit/proxy/test_usage_cost_egress.py +++ /dev/null @@ -1,200 +0,0 @@ -from __future__ import annotations - -from typing import Final - -import pytest - -import litellm -from litellm.proxy.usage_cost_egress import with_client_stream_usage_cost, with_client_usage_cost -from litellm.types.llms.openai import ( - ResponseAPIUsage, - ResponseCompletedEvent, - ResponseIncompleteEvent, - ResponsesAPIResponse, - ResponsesAPIStreamEvents, -) -from litellm.types.utils import Choices, Message, ModelResponse, ModelResponseStream, StreamingChoices, Usage - -UPSTREAM_COST: Final = 0.037 -OPERATOR_BILL: Final = 100 * 2e-6 + 20 * 6e-6 - - -def _chat_reply(upstream_cost: float | None) -> ModelResponse: - reply: Final = ModelResponse( - id="chatcmpl-egress", - model="omni-model", - choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], - usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=upstream_cost), - ) - reply._hidden_params["response_cost"] = OPERATOR_BILL - return reply - - -def _responses_reply(upstream_cost: float | None) -> ResponsesAPIResponse: - return ResponsesAPIResponse( - id="resp_egress", - created_at=1, - status="completed", - model="omni-model", - object="response", - output=[], - usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=upstream_cost), - ) - - -def _usage_chunk(upstream_cost: float | None) -> ModelResponseStream: - return ModelResponseStream( - id="chatcmpl-egress", - model="omni-model", - choices=[], - usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=upstream_cost), - ) - - -def _bill(_reply: object) -> float | None: - return OPERATOR_BILL - - -def _unpriceable(_reply: object) -> float | None: - return None - - -@pytest.mark.parametrize("reply", [_chat_reply(UPSTREAM_COST), _responses_reply(UPSTREAM_COST)]) -def test_a_reply_whose_upstream_reported_a_cost_tells_the_client_what_litellm_billed(reply: object) -> None: - billed: Final = with_client_usage_cost(reply, OPERATOR_BILL) - - assert billed is not reply, "the pipeline's own object keeps the upstream figure" - assert getattr(getattr(billed, "usage", None), "cost", None) == pytest.approx(OPERATOR_BILL) - assert getattr(getattr(reply, "usage", None), "cost", None) == UPSTREAM_COST - - -def test_the_billed_copy_of_a_chat_reply_keeps_its_hidden_params() -> None: - reply: Final = _chat_reply(UPSTREAM_COST) - - billed: Final = with_client_usage_cost(reply, OPERATOR_BILL) - - assert isinstance(billed, ModelResponse) - assert billed._hidden_params["response_cost"] == pytest.approx(OPERATOR_BILL) - assert billed.model_dump()["usage"]["cost"] == pytest.approx(OPERATOR_BILL) - - -@pytest.mark.parametrize("reply", [_chat_reply(None), _responses_reply(None)]) -def test_a_reply_without_an_upstream_cost_goes_out_as_the_upstream_sent_it( - monkeypatch: pytest.MonkeyPatch, reply: object -) -> None: - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) - - assert with_client_usage_cost(reply, OPERATOR_BILL) is reply - - -@pytest.mark.parametrize("reply", [_chat_reply(None), _responses_reply(None)]) -def test_include_cost_in_streaming_usage_puts_the_bill_on_a_reply_without_an_upstream_cost( - monkeypatch: pytest.MonkeyPatch, reply: object -) -> None: - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) - - billed: Final = with_client_usage_cost(reply, OPERATOR_BILL) - - assert getattr(getattr(billed, "usage", None), "cost", None) == pytest.approx(OPERATOR_BILL) - - -def test_a_zero_bill_replaces_the_upstream_cost() -> None: - billed: Final = with_client_usage_cost(_chat_reply(UPSTREAM_COST), 0.0) - - assert getattr(getattr(billed, "usage", None), "cost", None) == 0.0 - - -@pytest.mark.parametrize("bill", [None, -0.001, float("nan"), float("inf"), True, "0.1"]) -def test_an_unpriceable_reply_keeps_the_upstream_cost(bill: object) -> None: - reply: Final = _chat_reply(UPSTREAM_COST) - - assert with_client_usage_cost(reply, bill) is reply - assert reply.usage is not None and reply.usage.cost == UPSTREAM_COST - - -def test_a_reply_that_is_neither_chat_nor_responses_passes_through() -> None: - reply: Final = {"usage": {"cost": UPSTREAM_COST}} - - assert with_client_usage_cost(reply, OPERATOR_BILL) is reply - - -def test_the_streamed_usage_chunk_tells_the_client_what_litellm_billed() -> None: - chunk: Final = _usage_chunk(UPSTREAM_COST) - - billed: Final = with_client_stream_usage_cost(chunk, _bill) - - assert isinstance(billed, ModelResponseStream) and billed is not chunk - assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) - assert chunk.usage is not None and chunk.usage.cost == UPSTREAM_COST - - -def test_a_content_chunk_is_not_priced() -> None: - chunk: Final = ModelResponseStream( - id="chatcmpl-egress", model="omni-model", choices=[StreamingChoices(index=0, delta={"content": "hi"})] - ) - priced: list[object] = [] - - def price(reply: object) -> float | None: - priced.append(reply) - return OPERATOR_BILL - - assert with_client_stream_usage_cost(chunk, price) is chunk - assert priced == [] - - -def test_a_streamed_usage_chunk_without_an_upstream_cost_goes_out_as_sent(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) - chunk: Final = _usage_chunk(None) - - assert with_client_stream_usage_cost(chunk, _bill) is chunk - - -def test_include_cost_in_streaming_usage_puts_the_bill_on_the_streamed_usage_chunk( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) - - billed: Final = with_client_stream_usage_cost(_usage_chunk(None), _bill) - - assert isinstance(billed, ModelResponseStream) - assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) - - -def test_the_stream_pricer_sees_the_upstream_cost_so_the_bill_matches_the_spend_row() -> None: - seen: list[object] = [] - - def price(reply: object) -> float | None: - seen.append(getattr(getattr(reply, "usage", None), "cost", None)) - return OPERATOR_BILL - - with_client_stream_usage_cost(_usage_chunk(UPSTREAM_COST), price) - - assert seen == [UPSTREAM_COST] - - -@pytest.mark.parametrize("upstream_cost", [UPSTREAM_COST, None]) -@pytest.mark.parametrize( - "event_type", [ResponsesAPIStreamEvents.RESPONSE_COMPLETED, ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] -) -def test_a_terminal_responses_event_carries_the_bill_with_or_without_an_upstream_cost( - monkeypatch: pytest.MonkeyPatch, upstream_cost: float | None, event_type: ResponsesAPIStreamEvents -) -> None: - monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", False) - event_cls: Final = ( - ResponseCompletedEvent if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED else ResponseIncompleteEvent - ) - event: Final = event_cls(type=event_type, response=_responses_reply(upstream_cost)) - - billed: Final = with_client_stream_usage_cost(event, _bill) - - assert billed is not event - assert getattr(billed, "response").usage.cost == pytest.approx(OPERATOR_BILL) - assert event.response.usage is not None and event.response.usage.cost == upstream_cost - - -def test_an_unpriceable_terminal_responses_event_goes_out_as_sent() -> None: - event: Final = ResponseCompletedEvent( - type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_reply(None) - ) - - assert with_client_stream_usage_cost(event, _unpriceable) is event From 0a0281f1c11905e07e357a124c4671cd59c9e88f Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 27 Oct 2026 21:43:18 +0000 Subject: [PATCH 18/26] fix(proxy): a streamed usage.cost the client reads is what litellm billed, written once at the egress Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 20 +++- litellm/proxy/usage_cost_egress.py | 59 +++++++++ tests/unit/proxy/test_usage_cost_egress.py | 132 +++++++++++++++++++++ 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/usage_cost_egress.py create mode 100644 tests/unit/proxy/test_usage_cost_egress.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bc79becf7bb..5c7388f7861 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -922,6 +922,7 @@ from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import ( router as user_banner_endpoints_router, ) +from litellm.proxy.usage_cost_egress import Pricer, with_client_stream_usage_cost from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports PrismaClient, ProxyLogging, @@ -10134,6 +10135,22 @@ def _fast_serialize_simple_model_response_stream( return orjson.dumps(payload) +def _stream_usage_pricer(request_data: Mapping[str, object]) -> Pricer: + """Bills a usage-bearing stream event the way the request's logging object bills the spend row.""" + logging_obj: Final = request_data.get("litellm_logging_obj") + model_name: Final = str(request_data.get("model", "")) + + def price(reply: ModelResponse | ResponsesAPIResponse) -> float | None: + return ProxyBaseLLMRequestProcessing._streamed_usage_cost( # pyright: ignore[reportPrivateUsage] # the proxy's one stream pricer + reply, + model_name, + None, + logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + ) + + return price + + def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: if isinstance(chunk, ModelResponseStream): serialized_chunk: Final = _fast_serialize_simple_model_response_stream(chunk) @@ -10472,6 +10489,7 @@ async def async_data_generator( needs_per_chunk_hook: Final = proxy_logging_obj.needs_per_chunk_streaming_hook() is_raw_sse_stream: Final = bool(request_data.get("_litellm_raw_sse_stream")) strip_stream_usage: Final = bool(request_data.get("_litellm_strip_stream_usage")) + price_stream_usage: Final = _stream_usage_pricer(request_data) raw_sse_buffer = "" # A stream can start on a deployment with keepalive off and fall back @@ -10558,7 +10576,7 @@ async def async_data_generator( error_state.observe_chunk(chunk) raw_passthrough = False if isinstance(chunk, BaseModel): - chunk = _serialize_streaming_chunk(chunk) + chunk = _serialize_streaming_chunk(with_client_stream_usage_cost(chunk, price_stream_usage)) elif isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="replace") if is_raw_sse_stream: diff --git a/litellm/proxy/usage_cost_egress.py b/litellm/proxy/usage_cost_egress.py new file mode 100644 index 00000000000..b76d6579a6a --- /dev/null +++ b/litellm/proxy/usage_cost_egress.py @@ -0,0 +1,59 @@ +"""What a client reads in a streamed ``usage.cost``: litellm's bill, written once on the way out of the proxy, onto +a copy, so the final usage frame and the spend row agree. Inside the pipeline ``usage.cost`` stays what the +upstream reported.""" + +from __future__ import annotations + +import math +from collections.abc import Callable +from typing import Final, TypeVar + +from pydantic import BaseModel + +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, ModelResponseStream, Usage + +Pricer = Callable[[ModelResponse | ResponsesAPIResponse], float | None] +"""Prices a reply the way the request's logging object bills it (deployment rates, discount, margin).""" + +_RESPONSES_USAGE_EVENTS: Final = frozenset({"response.completed", "response.incomplete"}) +_ReplyT = TypeVar("_ReplyT", bound=BaseModel) + + +def _billable(cost: object) -> float | None: + if isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0: + return None + return float(cost) + + +def _client_usage_cost(upstream_cost: object, bill: object, *, wanted: bool) -> float | None: + """The bill, when the upstream put a number in usage.cost or the event carries one regardless (``wanted``). + None leaves the usage as the upstream sent it.""" + reported: Final = isinstance(upstream_cost, (int, float)) and not isinstance(upstream_cost, bool) + return _billable(bill) if reported or wanted else None + + +def _with_usage_cost(reply: _ReplyT, usage: Usage | ResponseAPIUsage, cost: float) -> _ReplyT: + return reply.model_copy(update={"usage": usage.model_copy(update={"cost": cost})}) + + +def with_client_stream_usage_cost(chunk: BaseModel, price: Pricer) -> BaseModel: + """A streamed event as the client gets it. Only the usage-bearing events are priced: the chat chunk carrying + ``usage``, rewritten where the upstream reported a cost, and the Responses ``response.completed`` / + ``response.incomplete`` events, which carry the bill whether or not the upstream reported one.""" + if isinstance(chunk, ModelResponseStream): + usage: Final = getattr(chunk, "usage", None) + if not isinstance(usage, Usage): + return chunk + priced: Final = ModelResponse(usage=usage, service_tier=getattr(chunk, "service_tier", None)) + cost: Final = _client_usage_cost(getattr(usage, "cost", None), price(priced), wanted=False) + return chunk if cost is None else _with_usage_cost(chunk, usage, cost) + if getattr(chunk, "type", None) not in _RESPONSES_USAGE_EVENTS: + return chunk + response: Final = getattr(chunk, "response", None) + if not isinstance(response, ResponsesAPIResponse) or not isinstance(response.usage, ResponseAPIUsage): + return chunk + response_cost: Final = _client_usage_cost(getattr(response.usage, "cost", None), price(response), wanted=True) + if response_cost is None: + return chunk + return chunk.model_copy(update={"response": _with_usage_cost(response, response.usage, response_cost)}) diff --git a/tests/unit/proxy/test_usage_cost_egress.py b/tests/unit/proxy/test_usage_cost_egress.py new file mode 100644 index 00000000000..0be4a6b745a --- /dev/null +++ b/tests/unit/proxy/test_usage_cost_egress.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from litellm.proxy.usage_cost_egress import with_client_stream_usage_cost +from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseIncompleteEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from litellm.types.utils import ModelResponseStream, StreamingChoices, Usage + +UPSTREAM_COST: Final = 0.037 +OPERATOR_BILL: Final = 100 * 2e-6 + 20 * 6e-6 + + +def _responses_reply(upstream_cost: float | None) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_egress", + created_at=1, + status="completed", + model="omni-model", + object="response", + output=[], + usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=upstream_cost), + ) + + +def _usage_chunk(upstream_cost: float | None, service_tier: str | None = None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-egress", + model="omni-model", + choices=[], + service_tier=service_tier, + usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=upstream_cost), + ) + + +def _bill(_reply: object) -> float | None: + return OPERATOR_BILL + + +def _unpriceable(_reply: object) -> float | None: + return None + + +def test_the_streamed_usage_chunk_tells_the_client_what_litellm_billed() -> None: + chunk: Final = _usage_chunk(UPSTREAM_COST) + + billed: Final = with_client_stream_usage_cost(chunk, _bill) + + assert isinstance(billed, ModelResponseStream) and billed is not chunk + assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) + assert chunk.usage is not None and chunk.usage.cost == UPSTREAM_COST + + +def test_a_zero_bill_replaces_the_upstream_cost() -> None: + billed: Final = with_client_stream_usage_cost(_usage_chunk(UPSTREAM_COST), lambda _reply: 0.0) + + assert isinstance(billed, ModelResponseStream) + assert billed.usage is not None and billed.usage.cost == 0.0 + + +@pytest.mark.parametrize("bill", [None, -0.001, float("nan"), float("inf"), True]) +def test_an_unpriceable_stream_keeps_the_upstream_cost(bill: float | bool | None) -> None: + chunk: Final = _usage_chunk(UPSTREAM_COST) + + assert with_client_stream_usage_cost(chunk, lambda _reply: bill) is chunk + assert chunk.usage is not None and chunk.usage.cost == UPSTREAM_COST + + +def test_a_streamed_usage_chunk_without_an_upstream_cost_goes_out_as_sent() -> None: + chunk: Final = _usage_chunk(None) + + assert with_client_stream_usage_cost(chunk, _bill) is chunk + + +def test_a_content_chunk_is_not_priced() -> None: + chunk: Final = ModelResponseStream( + id="chatcmpl-egress", model="omni-model", choices=[StreamingChoices(index=0, delta={"content": "hi"})] + ) + priced: list[object] = [] + + def price(reply: object) -> float | None: + priced.append(reply) + return OPERATOR_BILL + + assert with_client_stream_usage_cost(chunk, price) is chunk + assert priced == [] + + +def test_the_stream_pricer_sees_the_upstream_cost_and_the_served_tier_so_the_bill_matches_the_spend_row() -> None: + seen: list[tuple[object, object]] = [] + + def price(reply: object) -> float | None: + seen.append((getattr(getattr(reply, "usage", None), "cost", None), getattr(reply, "service_tier", None))) + return OPERATOR_BILL + + with_client_stream_usage_cost(_usage_chunk(UPSTREAM_COST, service_tier="default"), price) + + assert seen == [(UPSTREAM_COST, "default")] + + +@pytest.mark.parametrize("upstream_cost", [UPSTREAM_COST, None]) +@pytest.mark.parametrize( + "event_type", [ResponsesAPIStreamEvents.RESPONSE_COMPLETED, ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE] +) +def test_a_terminal_responses_event_carries_the_bill_with_or_without_an_upstream_cost( + upstream_cost: float | None, event_type: ResponsesAPIStreamEvents +) -> None: + event_cls: Final = ( + ResponseCompletedEvent if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED else ResponseIncompleteEvent + ) + event: Final = event_cls(type=event_type, response=_responses_reply(upstream_cost)) + + billed: Final = with_client_stream_usage_cost(event, _bill) + + assert billed is not event + assert getattr(billed, "response").usage.cost == pytest.approx(OPERATOR_BILL) + assert event.response.usage is not None and event.response.usage.cost == upstream_cost + + +def test_an_unpriceable_terminal_responses_event_goes_out_as_sent() -> None: + event: Final = ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_reply(None) + ) + + assert with_client_stream_usage_cost(event, _unpriceable) is event From 9ef0b5569e3964180047bd332f7dbc407973c1cf Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 27 Oct 2026 21:50:20 +0000 Subject: [PATCH 19/26] style(proxy): annotate the egress type alias and type var Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/usage_cost_egress.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/usage_cost_egress.py b/litellm/proxy/usage_cost_egress.py index b76d6579a6a..827c86e1ea7 100644 --- a/litellm/proxy/usage_cost_egress.py +++ b/litellm/proxy/usage_cost_egress.py @@ -6,18 +6,18 @@ from __future__ import annotations import math from collections.abc import Callable -from typing import Final, TypeVar +from typing import Final, TypeAlias, TypeVar from pydantic import BaseModel from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ModelResponse, ModelResponseStream, Usage -Pricer = Callable[[ModelResponse | ResponsesAPIResponse], float | None] +Pricer: TypeAlias = Callable[[ModelResponse | ResponsesAPIResponse], float | None] """Prices a reply the way the request's logging object bills it (deployment rates, discount, margin).""" _RESPONSES_USAGE_EVENTS: Final = frozenset({"response.completed", "response.incomplete"}) -_ReplyT = TypeVar("_ReplyT", bound=BaseModel) +_ReplyT: Final = TypeVar("_ReplyT", bound=BaseModel) def _billable(cost: object) -> float | None: From 68af9f061c0c66bbe83fbab34dd1259622cff752 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 27 Oct 2026 21:54:42 +0000 Subject: [PATCH 20/26] test(proxy): type the unpriceable stream bills as floats Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/proxy/test_usage_cost_egress.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/proxy/test_usage_cost_egress.py b/tests/unit/proxy/test_usage_cost_egress.py index 0be4a6b745a..bca4b94c46b 100644 --- a/tests/unit/proxy/test_usage_cost_egress.py +++ b/tests/unit/proxy/test_usage_cost_egress.py @@ -65,8 +65,8 @@ def test_a_zero_bill_replaces_the_upstream_cost() -> None: assert billed.usage is not None and billed.usage.cost == 0.0 -@pytest.mark.parametrize("bill", [None, -0.001, float("nan"), float("inf"), True]) -def test_an_unpriceable_stream_keeps_the_upstream_cost(bill: float | bool | None) -> None: +@pytest.mark.parametrize("bill", [None, -0.001, float("nan"), float("inf")]) +def test_an_unpriceable_stream_keeps_the_upstream_cost(bill: float | None) -> None: chunk: Final = _usage_chunk(UPSTREAM_COST) assert with_client_stream_usage_cost(chunk, lambda _reply: bill) is chunk From 09370b45dffd4749f48632a8b9aed04041fa4439 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 27 Oct 2026 21:59:24 +0000 Subject: [PATCH 21/26] fix(proxy): price a streamed Responses terminal event through the same stream pricer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 3b9c17baa02..26adbb8def5 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -273,6 +273,7 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( ModelResponse, ModelResponseStream, @@ -4724,7 +4725,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _completion_cost_or_none( - model_response: ModelResponse, model_name: str, service_tier: str | None + model_response: ModelResponse | ResponsesAPIResponse, model_name: str, service_tier: str | None ) -> float | None: try: return litellm.completion_cost( @@ -4735,7 +4736,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _logging_obj_cost_or_none( - model_response: ModelResponse, litellm_logging_obj: LiteLLMLoggingObj + model_response: ModelResponse | ResponsesAPIResponse, litellm_logging_obj: LiteLLMLoggingObj ) -> float | None: # Pricing a frame stamps cost_breakdown and, on failure, the cost-failure debug key onto # the live logging object. The pass-through handlers never recompute either one, so a @@ -4762,7 +4763,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _streamed_usage_cost( - model_response: ModelResponse, + model_response: ModelResponse | ResponsesAPIResponse, model_name: str, service_tier: str | None, litellm_logging_obj: LiteLLMLoggingObj | None, From 55dcb8f8c1e508fe253ecafd157d46fa95c05912 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 28 Oct 2026 00:24:32 +0000 Subject: [PATCH 22/26] fix(proxy): keep the bill in a priced chat stream's usage.cost when the upstream reported none Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/usage_cost_egress.py | 18 ++++++++++-------- tests/unit/proxy/test_usage_cost_egress.py | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/usage_cost_egress.py b/litellm/proxy/usage_cost_egress.py index 827c86e1ea7..817c8372610 100644 --- a/litellm/proxy/usage_cost_egress.py +++ b/litellm/proxy/usage_cost_egress.py @@ -26,11 +26,14 @@ def _billable(cost: object) -> float | None: return float(cost) -def _client_usage_cost(upstream_cost: object, bill: object, *, wanted: bool) -> float | None: - """The bill, when the upstream put a number in usage.cost or the event carries one regardless (``wanted``). - None leaves the usage as the upstream sent it.""" +def _client_usage_cost(upstream_cost: object, bill: object) -> float | None: + """The bill, where the upstream put a number in usage.cost (a $0 bill included) or litellm priced the + stream above zero. None leaves the usage as the upstream sent it.""" + billable: Final = _billable(bill) + if billable is None: + return None reported: Final = isinstance(upstream_cost, (int, float)) and not isinstance(upstream_cost, bool) - return _billable(bill) if reported or wanted else None + return billable if reported or billable > 0 else None def _with_usage_cost(reply: _ReplyT, usage: Usage | ResponseAPIUsage, cost: float) -> _ReplyT: @@ -39,21 +42,20 @@ def _with_usage_cost(reply: _ReplyT, usage: Usage | ResponseAPIUsage, cost: floa def with_client_stream_usage_cost(chunk: BaseModel, price: Pricer) -> BaseModel: """A streamed event as the client gets it. Only the usage-bearing events are priced: the chat chunk carrying - ``usage``, rewritten where the upstream reported a cost, and the Responses ``response.completed`` / - ``response.incomplete`` events, which carry the bill whether or not the upstream reported one.""" + ``usage`` and the Responses ``response.completed`` / ``response.incomplete`` events.""" if isinstance(chunk, ModelResponseStream): usage: Final = getattr(chunk, "usage", None) if not isinstance(usage, Usage): return chunk priced: Final = ModelResponse(usage=usage, service_tier=getattr(chunk, "service_tier", None)) - cost: Final = _client_usage_cost(getattr(usage, "cost", None), price(priced), wanted=False) + cost: Final = _client_usage_cost(getattr(usage, "cost", None), price(priced)) return chunk if cost is None else _with_usage_cost(chunk, usage, cost) if getattr(chunk, "type", None) not in _RESPONSES_USAGE_EVENTS: return chunk response: Final = getattr(chunk, "response", None) if not isinstance(response, ResponsesAPIResponse) or not isinstance(response.usage, ResponseAPIUsage): return chunk - response_cost: Final = _client_usage_cost(getattr(response.usage, "cost", None), price(response), wanted=True) + response_cost: Final = _client_usage_cost(getattr(response.usage, "cost", None), price(response)) if response_cost is None: return chunk return chunk.model_copy(update={"response": _with_usage_cost(response, response.usage, response_cost)}) diff --git a/tests/unit/proxy/test_usage_cost_egress.py b/tests/unit/proxy/test_usage_cost_egress.py index bca4b94c46b..cab33035443 100644 --- a/tests/unit/proxy/test_usage_cost_egress.py +++ b/tests/unit/proxy/test_usage_cost_egress.py @@ -73,10 +73,20 @@ def test_an_unpriceable_stream_keeps_the_upstream_cost(bill: float | None) -> No assert chunk.usage is not None and chunk.usage.cost == UPSTREAM_COST -def test_a_streamed_usage_chunk_without_an_upstream_cost_goes_out_as_sent() -> None: +def test_a_priced_stream_without_an_upstream_cost_still_tells_the_client_the_bill() -> None: chunk: Final = _usage_chunk(None) - assert with_client_stream_usage_cost(chunk, _bill) is chunk + billed: Final = with_client_stream_usage_cost(chunk, _bill) + + assert isinstance(billed, ModelResponseStream) and billed is not chunk + assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) + + +@pytest.mark.parametrize("bill", [None, 0.0]) +def test_an_unpriced_stream_without_an_upstream_cost_goes_out_as_sent(bill: float | None) -> None: + chunk: Final = _usage_chunk(None) + + assert with_client_stream_usage_cost(chunk, lambda _reply: bill) is chunk def test_a_content_chunk_is_not_priced() -> None: From b55caf052bd5b498c47ebb06209acedbb8d926b4 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 28 Oct 2026 01:10:26 +0000 Subject: [PATCH 23/26] fix(proxy): price the streamed usage frame from the served model, as the spend row is Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 4 +++- litellm/proxy/usage_cost_egress.py | 4 +++- .../test_streaming_handler.py | 22 +++++++++++++++++++ tests/unit/proxy/test_usage_cost_egress.py | 20 ++++++++++++----- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 8fb602ebeb0..6bd0e45fbc0 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2500,7 +2500,9 @@ class CustomStreamWrapper: return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS) def _new_usage_chunk(self) -> ModelResponseStream: - usage_chunk: Final = self.model_response_creator() + usage_chunk: Final = self.model_response_creator( + hidden_params=_provider_hidden_params(None, self._provider_response_model) + ) if self.caller_requested_stream_usage: usage_chunk.choices = [] return usage_chunk diff --git a/litellm/proxy/usage_cost_egress.py b/litellm/proxy/usage_cost_egress.py index 817c8372610..1dcdd69706f 100644 --- a/litellm/proxy/usage_cost_egress.py +++ b/litellm/proxy/usage_cost_egress.py @@ -10,6 +10,7 @@ from typing import Final, TypeAlias, TypeVar from pydantic import BaseModel +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import ModelResponse, ModelResponseStream, Usage @@ -47,7 +48,8 @@ def with_client_stream_usage_cost(chunk: BaseModel, price: Pricer) -> BaseModel: usage: Final = getattr(chunk, "usage", None) if not isinstance(usage, Usage): return chunk - priced: Final = ModelResponse(usage=usage, service_tier=getattr(chunk, "service_tier", None)) + priced: Final = ModelResponse(model=chunk.model, usage=usage, service_tier=getattr(chunk, "service_tier", None)) + priced.hidden_params = dict(get_hidden_params(chunk) or {}) cost: Final = _client_usage_cost(getattr(usage, "cost", None), price(priced)) return chunk if cost is None else _with_usage_cost(chunk, usage, cost) if getattr(chunk, "type", None) not in _RESPONSES_USAGE_EVENTS: diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 8d2b4b51164..6ccb4b8828c 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -6118,6 +6118,28 @@ def test_chunk_creator_keeps_provider_model_private_across_stream(): ) +def test_the_usage_chunk_carries_the_served_model_so_its_price_matches_the_assembled_response(): + from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict + + wrapper = CustomStreamWrapper( + completion_stream=None, + model="requested-route", + logging_obj=MagicMock(), + custom_llm_provider="openai", + ) + wrapper.chunk_creator( + chunk=ModelResponseStream( + id="chunk-1", + model="selected-model", + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="hello"))], + ) + ) + + usage_chunk = wrapper._new_usage_chunk() + + assert get_hidden_params_dict(usage_chunk)["provider_response_model"] == "selected-model" + + def test_assembled_stream_uses_later_provider_model_for_cost( monkeypatch: pytest.MonkeyPatch, ): diff --git a/tests/unit/proxy/test_usage_cost_egress.py b/tests/unit/proxy/test_usage_cost_egress.py index cab33035443..070139c5019 100644 --- a/tests/unit/proxy/test_usage_cost_egress.py +++ b/tests/unit/proxy/test_usage_cost_egress.py @@ -103,16 +103,26 @@ def test_a_content_chunk_is_not_priced() -> None: assert priced == [] -def test_the_stream_pricer_sees_the_upstream_cost_and_the_served_tier_so_the_bill_matches_the_spend_row() -> None: - seen: list[tuple[object, object]] = [] +def test_the_stream_pricer_sees_the_served_model_tier_and_upstream_cost_so_the_bill_matches_the_spend_row() -> None: + seen: list[tuple[object, object, object, object]] = [] def price(reply: object) -> float | None: - seen.append((getattr(getattr(reply, "usage", None), "cost", None), getattr(reply, "service_tier", None))) + seen.append( + ( + getattr(reply, "model", None), + getattr(reply, "hidden_params", {}).get("provider_response_model"), + getattr(reply, "service_tier", None), + getattr(getattr(reply, "usage", None), "cost", None), + ) + ) return OPERATOR_BILL - with_client_stream_usage_cost(_usage_chunk(UPSTREAM_COST, service_tier="default"), price) + chunk: Final = _usage_chunk(UPSTREAM_COST, service_tier="default") + chunk.hidden_params = {"provider_response_model": "gpt-4o-mini"} - assert seen == [(UPSTREAM_COST, "default")] + with_client_stream_usage_cost(chunk, price) + + assert seen == [("omni-model", "gpt-4o-mini", "default", UPSTREAM_COST)] @pytest.mark.parametrize("upstream_cost", [UPSTREAM_COST, None]) From b6571348e577c726cd8201c546a082c52cf39c5e Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 28 Oct 2026 02:02:31 +0000 Subject: [PATCH 24/26] refactor(proxy): price a stream once and copy the builder's bill into the streamed usage.cost at egress The egress no longer re-prices a reconstructed chunk. The stream wrapper copies the hidden response_cost the builder computed for the spend row onto the usage chunk, the Responses iterator stamps the logging object's bill the same way, and the proxy egress only copies that number into the client-visible usage.cost. Deletes the egress pricer, the _streamed_usage_cost widening and the served-model carry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/streaming_handler.py | 17 ++- litellm/proxy/common_request_processing.py | 7 +- litellm/proxy/proxy_server.py | 21 +--- litellm/proxy/usage_cost_egress.py | 32 ++--- .../streaming_iterator.py | 2 +- litellm/responses/streaming_iterator.py | 21 +++- .../test_streaming_handler.py | 53 ++++----- tests/unit/proxy/test_usage_cost_egress.py | 112 +++++++----------- .../test_streaming_iterator_transformation.py | 28 +++++ .../unit/responses/test_streaming_iterator.py | 24 +++- 10 files changed, 167 insertions(+), 150 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6bd0e45fbc0..4497f3de9f3 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -92,6 +92,17 @@ def _next_sync_or_exhausted(it: Iterator[object]) -> object: return _SYNC_ITER_EXHAUSTED +def _stamp_hidden_response_cost( + usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream, + complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream, +) -> None: + """The bill the builder priced for the spend row rides the usage chunk, so the proxy shows the same number.""" + bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost") + target: Final = get_hidden_params(usage_chunk) + if target is not None and isinstance(bill, (int, float)) and not isinstance(bill, bool): + target["response_cost"] = float(bill) + + def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None: served_tier: Final = complete_streaming_response.model_dump().get("service_tier") if isinstance(served_tier, str) and served_tier: @@ -2122,6 +2133,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_hidden_response_cost(response, complete_streaming_response) try: _cache_copy = complete_streaming_response.model_copy(deep=True) _log_copy = complete_streaming_response.model_copy(deep=True) @@ -2343,6 +2355,7 @@ class CustomStreamWrapper: getattr(complete_streaming_response, "usage"), ) _stamp_served_service_tier(response, complete_streaming_response) + _stamp_hidden_response_cost(response, complete_streaming_response) try: _copy = complete_streaming_response.model_copy(deep=True) except RuntimeError: @@ -2500,9 +2513,7 @@ class CustomStreamWrapper: return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS) def _new_usage_chunk(self) -> ModelResponseStream: - usage_chunk: Final = self.model_response_creator( - hidden_params=_provider_hidden_params(None, self._provider_response_model) - ) + usage_chunk: Final = self.model_response_creator() if self.caller_requested_stream_usage: usage_chunk.choices = [] return usage_chunk diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 26adbb8def5..3b9c17baa02 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -273,7 +273,6 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD -from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( ModelResponse, ModelResponseStream, @@ -4725,7 +4724,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _completion_cost_or_none( - model_response: ModelResponse | ResponsesAPIResponse, model_name: str, service_tier: str | None + model_response: ModelResponse, model_name: str, service_tier: str | None ) -> float | None: try: return litellm.completion_cost( @@ -4736,7 +4735,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _logging_obj_cost_or_none( - model_response: ModelResponse | ResponsesAPIResponse, litellm_logging_obj: LiteLLMLoggingObj + model_response: ModelResponse, litellm_logging_obj: LiteLLMLoggingObj ) -> float | None: # Pricing a frame stamps cost_breakdown and, on failure, the cost-failure debug key onto # the live logging object. The pass-through handlers never recompute either one, so a @@ -4763,7 +4762,7 @@ class ProxyBaseLLMRequestProcessing: @staticmethod def _streamed_usage_cost( - model_response: ModelResponse | ResponsesAPIResponse, + model_response: ModelResponse, model_name: str, service_tier: str | None, litellm_logging_obj: LiteLLMLoggingObj | None, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5c7388f7861..87d90904ee2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -922,7 +922,7 @@ from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( from litellm.proxy.ui_crud_endpoints.user_banner_endpoints import ( router as user_banner_endpoints_router, ) -from litellm.proxy.usage_cost_egress import Pricer, with_client_stream_usage_cost +from litellm.proxy.usage_cost_egress import with_client_stream_usage_cost from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports PrismaClient, ProxyLogging, @@ -10135,22 +10135,6 @@ def _fast_serialize_simple_model_response_stream( return orjson.dumps(payload) -def _stream_usage_pricer(request_data: Mapping[str, object]) -> Pricer: - """Bills a usage-bearing stream event the way the request's logging object bills the spend row.""" - logging_obj: Final = request_data.get("litellm_logging_obj") - model_name: Final = str(request_data.get("model", "")) - - def price(reply: ModelResponse | ResponsesAPIResponse) -> float | None: - return ProxyBaseLLMRequestProcessing._streamed_usage_cost( # pyright: ignore[reportPrivateUsage] # the proxy's one stream pricer - reply, - model_name, - None, - logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, - ) - - return price - - def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes: if isinstance(chunk, ModelResponseStream): serialized_chunk: Final = _fast_serialize_simple_model_response_stream(chunk) @@ -10489,7 +10473,6 @@ async def async_data_generator( needs_per_chunk_hook: Final = proxy_logging_obj.needs_per_chunk_streaming_hook() is_raw_sse_stream: Final = bool(request_data.get("_litellm_raw_sse_stream")) strip_stream_usage: Final = bool(request_data.get("_litellm_strip_stream_usage")) - price_stream_usage: Final = _stream_usage_pricer(request_data) raw_sse_buffer = "" # A stream can start on a deployment with keepalive off and fall back @@ -10576,7 +10559,7 @@ async def async_data_generator( error_state.observe_chunk(chunk) raw_passthrough = False if isinstance(chunk, BaseModel): - chunk = _serialize_streaming_chunk(with_client_stream_usage_cost(chunk, price_stream_usage)) + chunk = _serialize_streaming_chunk(with_client_stream_usage_cost(chunk)) elif isinstance(chunk, bytes): chunk = chunk.decode("utf-8", errors="replace") if is_raw_sse_stream: diff --git a/litellm/proxy/usage_cost_egress.py b/litellm/proxy/usage_cost_egress.py index 1dcdd69706f..ac29c1cef09 100644 --- a/litellm/proxy/usage_cost_egress.py +++ b/litellm/proxy/usage_cost_egress.py @@ -1,21 +1,17 @@ -"""What a client reads in a streamed ``usage.cost``: litellm's bill, written once on the way out of the proxy, onto -a copy, so the final usage frame and the spend row agree. Inside the pipeline ``usage.cost`` stays what the -upstream reported.""" +"""What a client reads in a streamed ``usage.cost``: the bill litellm already priced for the spend row, copied once on +the way out of the proxy onto a copy of the event. Nothing is priced here. Inside the pipeline ``usage.cost`` stays +what the upstream reported.""" from __future__ import annotations import math -from collections.abc import Callable -from typing import Final, TypeAlias, TypeVar +from typing import Final, TypeVar from pydantic import BaseModel from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse -from litellm.types.utils import ModelResponse, ModelResponseStream, Usage - -Pricer: TypeAlias = Callable[[ModelResponse | ResponsesAPIResponse], float | None] -"""Prices a reply the way the request's logging object bills it (deployment rates, discount, margin).""" +from litellm.types.utils import ModelResponseStream, Usage _RESPONSES_USAGE_EVENTS: Final = frozenset({"response.completed", "response.incomplete"}) _ReplyT: Final = TypeVar("_ReplyT", bound=BaseModel) @@ -41,23 +37,27 @@ def _with_usage_cost(reply: _ReplyT, usage: Usage | ResponseAPIUsage, cost: floa return reply.model_copy(update={"usage": usage.model_copy(update={"cost": cost})}) -def with_client_stream_usage_cost(chunk: BaseModel, price: Pricer) -> BaseModel: - """A streamed event as the client gets it. Only the usage-bearing events are priced: the chat chunk carrying - ``usage`` and the Responses ``response.completed`` / ``response.incomplete`` events.""" +def _hidden_bill(reply: object) -> object: + hidden: Final = get_hidden_params(reply) + return hidden.get("response_cost") if hidden is not None else None + + +def with_client_stream_usage_cost(chunk: BaseModel) -> BaseModel: + """A streamed event as the client gets it. Only the usage-bearing events change: the chat chunk carrying + ``usage`` and the Responses ``response.completed`` / ``response.incomplete`` events, each read against the + hidden ``response_cost`` the pipeline stamped on it.""" if isinstance(chunk, ModelResponseStream): usage: Final = getattr(chunk, "usage", None) if not isinstance(usage, Usage): return chunk - priced: Final = ModelResponse(model=chunk.model, usage=usage, service_tier=getattr(chunk, "service_tier", None)) - priced.hidden_params = dict(get_hidden_params(chunk) or {}) - cost: Final = _client_usage_cost(getattr(usage, "cost", None), price(priced)) + cost: Final = _client_usage_cost(getattr(usage, "cost", None), _hidden_bill(chunk)) return chunk if cost is None else _with_usage_cost(chunk, usage, cost) if getattr(chunk, "type", None) not in _RESPONSES_USAGE_EVENTS: return chunk response: Final = getattr(chunk, "response", None) if not isinstance(response, ResponsesAPIResponse) or not isinstance(response.usage, ResponseAPIUsage): return chunk - response_cost: Final = _client_usage_cost(getattr(response.usage, "cost", None), price(response)) + response_cost: Final = _client_usage_cost(getattr(response.usage, "cost", None), _hidden_bill(response)) if response_cost is None: return chunk return chunk.model_copy(update={"response": _with_usage_cost(response, response.usage, response_cost)}) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index dc84a955d88..da375b7265c 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1480,7 +1480,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): } ) ) - normalize_responses_usage(partial) + normalize_responses_usage(partial, self.litellm_logging_obj) return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) def _emit_terminal_response_event( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 9daf350baa7..11c8d5405c0 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,6 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -585,7 +586,7 @@ class BaseResponsesAPIStreamingIterator: else openai_responses_api_chunk.model_copy(update={"response": _billed_response}) ) self.completed_response = _terminal_chunk - normalize_responses_usage(_billed_response) + normalize_responses_usage(_billed_response, self.logging_obj) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() @@ -1069,7 +1070,7 @@ class BaseResponsesAPIStreamingIterator: "output": _partial_output_items(self._generated_content), } ) - normalize_responses_usage(partial) + normalize_responses_usage(partial, self.logging_obj) return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None: @@ -1741,14 +1742,24 @@ def _estimate_usage_safely( return None -def normalize_responses_usage(response_obj: ResponsesAPIResponse | None) -> None: - """A terminal response built off an untyped dict carries its usage as a dict, logging and the proxy read a model.""" +def normalize_responses_usage(response_obj: ResponsesAPIResponse | None, logging_obj: LiteLLMLoggingObj | None) -> None: + """A terminal response built off an untyped dict carries its usage as a dict, logging and the proxy read a model. + The bill the logging object prices for it rides its hidden ``response_cost``, so the proxy shows the same number.""" if response_obj is None: return usage_obj: Final[ResponseAPIUsage | None] = _usage_as_model(getattr(response_obj, "usage", None)) if usage_obj is None: return response_obj.usage = usage_obj # rebind-ok: the typed usage has to ride on the response the client receives + if logging_obj is None: + return + try: + bill: Final = logging_obj.response_cost_calculator(result=response_obj) + except Exception: # noqa: BLE001 # a pricing failure leaves the client usage as the upstream sent it + return + hidden: Final = get_hidden_params(response_obj) + if hidden is not None and isinstance(bill, (int, float)) and not isinstance(bill, bool): + hidden["response_cost"] = float(bill) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: @@ -1782,7 +1793,7 @@ def build_synthetic_response_events( chunk_size: int, ) -> list[ResponsesAPIStreamingResponse]: openai_types: Final = get_openai_response_types() - normalize_responses_usage(transformed) + normalize_responses_usage(transformed, logging_obj) events: Final[list[ResponsesAPIStreamingResponse]] = [ _build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed), diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 6ccb4b8828c..d24d147b3df 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -8,15 +8,16 @@ import time from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import CodeType, FrameType, MappingProxyType -from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast +from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, Tuple, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest from openai.types.chat import ChatCompletionChunk from pydantic import BaseModel, TypeAdapter + import litellm -from litellm import verbose_logger +from litellm import completion, verbose_logger from litellm._logging import session_id_var, trace_id_var from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger @@ -27,7 +28,6 @@ from litellm.litellm_core_utils.streaming_handler import ( _ProviderChunkEarlyReturn, _ProviderChunkParsed, ) -from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.responses.mcp.chat_completions_handler import _add_mcp_metadata_to_response from litellm.types.utils import ( CompletionTokensDetailsWrapper, @@ -35,13 +35,11 @@ from litellm.types.utils import ( ModelResponse, ModelResponseStream, PromptTokensDetailsWrapper, - StreamingChoices, StandardLoggingPayload, + StreamingChoices, Usage, ) from litellm.utils import ModelResponseListIterator -from litellm import completion -from typing import Tuple @pytest.mark.parametrize("async_stream", [False, True]) @@ -2436,7 +2434,7 @@ def test_openai_compatible_streaming_cost_is_the_reported_cost(): custom_llm_provider="openai", call_type="completion", optional_params={}, - ) == pytest.approx(0.037) + ) == 0.037 def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch): @@ -6059,6 +6057,25 @@ def test_chunk_creator_preserves_hidden_provider_specific_fields_from_parsed_chu assert assembled._hidden_params["provider_specific_fields"] == {"traffic_type": "ON_DEMAND_FLEX"} +def test_the_usage_chunk_carries_the_bill_the_builder_priced_for_the_spend_row() -> None: + from litellm.litellm_core_utils.hidden_params import get_hidden_params + from litellm.litellm_core_utils.streaming_handler import ( + _stamp_hidden_response_cost, # pyright: ignore[reportPrivateUsage] # the helper under test + ) + + usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + complete: Final = ModelResponse(model="omni-model", usage=usage) + complete.hidden_params = {"response_cost": 0.00032} + usage_chunk: Final = ModelResponseStream(id="chunk-1", model="omni-model", choices=[], usage=usage) + usage_chunk.hidden_params = {"response_cost": None} + + _stamp_hidden_response_cost(usage_chunk, complete) + + stamped: Final = get_hidden_params(usage_chunk) + assert stamped is not None and stamped["response_cost"] == 0.00032 + assert getattr(usage, "cost", None) is None + + def test_chunk_creator_keeps_provider_model_private_across_stream(): from litellm.router_utils.add_retry_fallback_headers import ( get_hidden_params_dict, @@ -6118,28 +6135,6 @@ def test_chunk_creator_keeps_provider_model_private_across_stream(): ) -def test_the_usage_chunk_carries_the_served_model_so_its_price_matches_the_assembled_response(): - from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict - - wrapper = CustomStreamWrapper( - completion_stream=None, - model="requested-route", - logging_obj=MagicMock(), - custom_llm_provider="openai", - ) - wrapper.chunk_creator( - chunk=ModelResponseStream( - id="chunk-1", - model="selected-model", - choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="hello"))], - ) - ) - - usage_chunk = wrapper._new_usage_chunk() - - assert get_hidden_params_dict(usage_chunk)["provider_response_model"] == "selected-model" - - def test_assembled_stream_uses_later_provider_model_for_cost( monkeypatch: pytest.MonkeyPatch, ): diff --git a/tests/unit/proxy/test_usage_cost_egress.py b/tests/unit/proxy/test_usage_cost_egress.py index 070139c5019..62c32cd3ebc 100644 --- a/tests/unit/proxy/test_usage_cost_egress.py +++ b/tests/unit/proxy/test_usage_cost_egress.py @@ -4,6 +4,7 @@ from typing import Final import pytest +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.proxy.usage_cost_egress import with_client_stream_usage_cost from litellm.types.llms.openai import ( ResponseAPIUsage, @@ -12,14 +13,14 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import ModelResponseStream, StreamingChoices, Usage +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage UPSTREAM_COST: Final = 0.037 OPERATOR_BILL: Final = 100 * 2e-6 + 20 * 6e-6 -def _responses_reply(upstream_cost: float | None) -> ResponsesAPIResponse: - return ResponsesAPIResponse( +def _responses_reply(upstream_cost: float | None, bill: object) -> ResponsesAPIResponse: + reply: Final = ResponsesAPIResponse( id="resp_egress", created_at=1, status="completed", @@ -28,101 +29,71 @@ def _responses_reply(upstream_cost: float | None) -> ResponsesAPIResponse: output=[], usage=ResponseAPIUsage(input_tokens=100, output_tokens=20, total_tokens=120, cost=upstream_cost), ) + hidden: Final = get_hidden_params(reply) + assert hidden is not None + hidden["response_cost"] = bill + return reply -def _usage_chunk(upstream_cost: float | None, service_tier: str | None = None) -> ModelResponseStream: - return ModelResponseStream( +def _usage_chunk(upstream_cost: float | None, bill: object) -> ModelResponseStream: + chunk: Final = ModelResponseStream( id="chatcmpl-egress", model="omni-model", choices=[], - service_tier=service_tier, usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=upstream_cost), ) - - -def _bill(_reply: object) -> float | None: - return OPERATOR_BILL - - -def _unpriceable(_reply: object) -> float | None: - return None + chunk.hidden_params = {"response_cost": bill} + return chunk def test_the_streamed_usage_chunk_tells_the_client_what_litellm_billed() -> None: - chunk: Final = _usage_chunk(UPSTREAM_COST) + chunk: Final = _usage_chunk(UPSTREAM_COST, OPERATOR_BILL) - billed: Final = with_client_stream_usage_cost(chunk, _bill) + billed: Final = with_client_stream_usage_cost(chunk) assert isinstance(billed, ModelResponseStream) and billed is not chunk - assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) + assert billed.usage is not None and billed.usage.cost == OPERATOR_BILL assert chunk.usage is not None and chunk.usage.cost == UPSTREAM_COST def test_a_zero_bill_replaces_the_upstream_cost() -> None: - billed: Final = with_client_stream_usage_cost(_usage_chunk(UPSTREAM_COST), lambda _reply: 0.0) + billed: Final = with_client_stream_usage_cost(_usage_chunk(UPSTREAM_COST, 0.0)) assert isinstance(billed, ModelResponseStream) assert billed.usage is not None and billed.usage.cost == 0.0 -@pytest.mark.parametrize("bill", [None, -0.001, float("nan"), float("inf")]) -def test_an_unpriceable_stream_keeps_the_upstream_cost(bill: float | None) -> None: - chunk: Final = _usage_chunk(UPSTREAM_COST) +@pytest.mark.parametrize("bill", [None, -0.001, float("nan"), float("inf"), True]) +def test_an_unpriced_stream_keeps_the_upstream_cost(bill: object) -> None: + chunk: Final = _usage_chunk(UPSTREAM_COST, bill) - assert with_client_stream_usage_cost(chunk, lambda _reply: bill) is chunk + assert with_client_stream_usage_cost(chunk) is chunk assert chunk.usage is not None and chunk.usage.cost == UPSTREAM_COST def test_a_priced_stream_without_an_upstream_cost_still_tells_the_client_the_bill() -> None: - chunk: Final = _usage_chunk(None) + billed: Final = with_client_stream_usage_cost(_usage_chunk(None, OPERATOR_BILL)) - billed: Final = with_client_stream_usage_cost(chunk, _bill) - - assert isinstance(billed, ModelResponseStream) and billed is not chunk - assert billed.usage is not None and billed.usage.cost == pytest.approx(OPERATOR_BILL) + assert isinstance(billed, ModelResponseStream) + assert billed.usage is not None and billed.usage.cost == OPERATOR_BILL @pytest.mark.parametrize("bill", [None, 0.0]) def test_an_unpriced_stream_without_an_upstream_cost_goes_out_as_sent(bill: float | None) -> None: - chunk: Final = _usage_chunk(None) + chunk: Final = _usage_chunk(None, bill) - assert with_client_stream_usage_cost(chunk, lambda _reply: bill) is chunk + assert with_client_stream_usage_cost(chunk) is chunk -def test_a_content_chunk_is_not_priced() -> None: +def test_a_content_chunk_is_not_touched() -> None: chunk: Final = ModelResponseStream( - id="chatcmpl-egress", model="omni-model", choices=[StreamingChoices(index=0, delta={"content": "hi"})] + id="chatcmpl-egress", + model="omni-model", + choices=[StreamingChoices(index=0, delta=Delta(content="hi"), finish_reason=None)], ) - priced: list[object] = [] + chunk.hidden_params = {"response_cost": OPERATOR_BILL} - def price(reply: object) -> float | None: - priced.append(reply) - return OPERATOR_BILL - - assert with_client_stream_usage_cost(chunk, price) is chunk - assert priced == [] - - -def test_the_stream_pricer_sees_the_served_model_tier_and_upstream_cost_so_the_bill_matches_the_spend_row() -> None: - seen: list[tuple[object, object, object, object]] = [] - - def price(reply: object) -> float | None: - seen.append( - ( - getattr(reply, "model", None), - getattr(reply, "hidden_params", {}).get("provider_response_model"), - getattr(reply, "service_tier", None), - getattr(getattr(reply, "usage", None), "cost", None), - ) - ) - return OPERATOR_BILL - - chunk: Final = _usage_chunk(UPSTREAM_COST, service_tier="default") - chunk.hidden_params = {"provider_response_model": "gpt-4o-mini"} - - with_client_stream_usage_cost(chunk, price) - - assert seen == [("omni-model", "gpt-4o-mini", "default", UPSTREAM_COST)] + assert with_client_stream_usage_cost(chunk) is chunk @pytest.mark.parametrize("upstream_cost", [UPSTREAM_COST, None]) @@ -132,21 +103,24 @@ def test_the_stream_pricer_sees_the_served_model_tier_and_upstream_cost_so_the_b def test_a_terminal_responses_event_carries_the_bill_with_or_without_an_upstream_cost( upstream_cost: float | None, event_type: ResponsesAPIStreamEvents ) -> None: - event_cls: Final = ( - ResponseCompletedEvent if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED else ResponseIncompleteEvent + reply: Final = _responses_reply(upstream_cost, OPERATOR_BILL) + event: Final = ( + ResponseCompletedEvent(type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=reply) + if event_type is ResponsesAPIStreamEvents.RESPONSE_COMPLETED + else ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=reply) ) - event: Final = event_cls(type=event_type, response=_responses_reply(upstream_cost)) - billed: Final = with_client_stream_usage_cost(event, _bill) + billed: Final = with_client_stream_usage_cost(event) assert billed is not event - assert getattr(billed, "response").usage.cost == pytest.approx(OPERATOR_BILL) - assert event.response.usage is not None and event.response.usage.cost == upstream_cost + assert isinstance(billed, (ResponseCompletedEvent, ResponseIncompleteEvent)) + assert billed.response.usage is not None and billed.response.usage.cost == OPERATOR_BILL + assert reply.usage is not None and reply.usage.cost == upstream_cost -def test_an_unpriceable_terminal_responses_event_goes_out_as_sent() -> None: +def test_an_unpriced_terminal_responses_event_goes_out_as_sent() -> None: event: Final = ResponseCompletedEvent( - type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_reply(None) + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=_responses_reply(UPSTREAM_COST, None) ) - assert with_client_stream_usage_cost(event, _unpriceable) is event + assert with_client_stream_usage_cost(event) is event diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index ed90acbd2fa..e4784e92bc4 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -10,6 +10,7 @@ before response.completed, and that every event of a bridged stream carries the spend tracking stores, so a follow-up previous_response_id still finds the conversation. """ +import itertools import json from collections.abc import Callable from typing import Final @@ -23,6 +24,7 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, + OutputTextDeltaEvent, ResponsesAPIStreamEvents, ) from litellm.types.responses.main import build_web_search_call @@ -1558,3 +1560,29 @@ async def test_a_compaction_block_without_its_summary_streams_no_compaction_item assert [event["item"]["type"] for event in events if "item" in event] == ["message", "message"] assert [item["type"] for item in events[-1]["response"]["output"]] == ["message"] + + +def test_the_partial_response_billed_on_disconnect_carries_the_logging_objects_bill_as_hidden_response_cost() -> None: + from litellm.litellm_core_utils.hidden_params import get_hidden_params + + wrapper: Final = _FakeStreamWrapper([_chunk("Hello"), _chunk(" world", finish_reason="stop")]) + wrapper.logging_obj.response_cost_calculator.return_value = 0.00032 + iterator: Final = LiteLLMCompletionStreamingIterator( + model="claude-haiku-4-5", + litellm_custom_stream_wrapper=wrapper, # pyright: ignore[reportArgumentType] # the fake stands in for CustomStreamWrapper, as in _build_iterator + request_input="What is the weather in San Francisco?", + responses_api_request={}, + custom_llm_provider="anthropic", + litellm_metadata={}, + ) + relayed: Final = tuple( + itertools.takewhile(lambda event: not isinstance(event, OutputTextDeltaEvent), iterator) + ) + assert relayed, "the stream must relay at least one chunk before the client goes away" + + partial: Final = iterator.partial_response_on_disconnect() + + assert partial is not None + hidden: Final = get_hidden_params(partial.response) + assert hidden is not None and hidden["response_cost"] == 0.00032 + assert wrapper.logging_obj.response_cost_calculator.call_args.kwargs["result"] is partial.response diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index bd9122a927c..710bcb43e25 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -25,6 +25,7 @@ from pydantic_core import PydanticSerializationError import litellm from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.responses import streaming_iterator as streaming_module @@ -864,7 +865,7 @@ def test_normalize_responses_usage_types_a_dict_usage_and_keeps_the_provider_cos } ) - normalize_responses_usage(response) + normalize_responses_usage(response, None) assert isinstance(response.usage, ResponseAPIUsage) assert response.usage.cost == pytest.approx(3e-05) @@ -876,11 +877,26 @@ def test_normalize_responses_usage_never_writes_a_cost_the_provider_did_not_repo response = _responses_api_response_with_usage() - normalize_responses_usage(response) + normalize_responses_usage(response, None) assert getattr(response.usage, "cost", None) is None +def test_normalize_responses_usage_stamps_the_logging_objects_bill_as_hidden_response_cost(): + from litellm.responses.streaming_iterator import normalize_responses_usage + + response = _responses_api_response_with_usage() + logging_obj = MagicMock() + logging_obj.response_cost_calculator.return_value = 0.00032 + + normalize_responses_usage(response, logging_obj) + + hidden: Final = get_hidden_params(response) + assert hidden is not None and hidden["response_cost"] == 0.00032 + assert getattr(response.usage, "cost", None) is None, "the client field is written at the proxy egress, not here" + assert logging_obj.response_cost_calculator.call_args.kwargs["result"] is response + + def _capture_dispatch(logged: list): """Record the object handed to the success handlers. @@ -898,7 +914,7 @@ def _capture_dispatch(logged: list): return _dispatch -def _headers_config(*, transform_hidden_params: dict | None = None) -> Mock: +def _headers_config(*, transform_hidden_params: Mapping[str, object] | None = None) -> Mock: """Config whose completed event carries a real ResponsesAPIResponse, so the logging copy performs a genuine model_dump/model_validate round trip.""" mock_config = Mock(spec=BaseResponsesAPIConfig) @@ -1683,7 +1699,7 @@ def test_process_chunk_wraps_encrypted_content_with_model_id(): @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") -def test_process_chunk_completed_response_updates_id_and_leaves_usage_cost_alone(monkeypatch): +def test_process_chunk_completed_response_updates_id_and_leaves_usage_cost_alone(monkeypatch: pytest.MonkeyPatch): original_include_cost = litellm.include_cost_in_streaming_usage litellm.include_cost_in_streaming_usage = True openai_types = streaming_module._get_openai_response_types() From e2033586a4ce2f29ea26f14d78c93745cea23b8a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 28 Oct 2026 02:28:17 +0000 Subject: [PATCH 25/26] fix(hidden_params): read private attributes only off a real pydantic class A spec'd test double passes isinstance(obj, BaseModel) through __class__ while its type has no __private_attributes__, which raised on every hidden params read Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/hidden_params.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index a74337d2549..202226fa9ad 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -63,7 +63,8 @@ def stored_attribute(obj: object, name: str) -> object: def _get_hidden_params_storage(obj: object) -> object | None: if isinstance(obj, dict): return obj.get(_HIDDEN_PARAMS_ATTR) - if isinstance(obj, BaseModel) and _HIDDEN_PARAMS_ATTR in type(obj).__private_attributes__: + model_type: Final = type(obj) + if issubclass(model_type, BaseModel) and _HIDDEN_PARAMS_ATTR in model_type.__private_attributes__: return getattr(obj, _HIDDEN_PARAMS_ATTR, None) return stored_attribute(obj, _HIDDEN_PARAMS_ATTR) From 856c22a1cd0c8fdd6497b4864448a8fb2a3af393 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 28 Oct 2026 03:10:24 +0000 Subject: [PATCH 26/26] fix(cost): keep a $0 stream bill in hidden response_cost so the egress shows it A free or fully discounted stream used to leave the upstream usage.cost on the client frame while the spend row read 0, because the builder skipped stamping a zero bill Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../streaming_chunk_builder_utils.py | 2 +- .../unit/responses/test_streaming_iterator.py | 23 ++++++++++++++---- tests/unit/test_main.py | 24 +++++++++++++++++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 7cf024be2a1..daa816179a1 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1415,7 +1415,7 @@ def _stamp_streaming_response_cost(response: ModelResponse, logging_obj: Optiona if logging_obj is None: return computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: + if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0: return _set_hidden_response_cost(response, float(computed_cost)) diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 710bcb43e25..854719783bc 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -885,16 +885,29 @@ def test_normalize_responses_usage_never_writes_a_cost_the_provider_did_not_repo def test_normalize_responses_usage_stamps_the_logging_objects_bill_as_hidden_response_cost(): from litellm.responses.streaming_iterator import normalize_responses_usage - response = _responses_api_response_with_usage() - logging_obj = MagicMock() - logging_obj.response_cost_calculator.return_value = 0.00032 + from litellm.litellm_core_utils.litellm_logging import Logging + + response: Final = _responses_api_response_with_usage() + logging_obj: Final = Logging( + model="mantle-claude", + messages=[{"role": "user", "content": "q"}], + stream=True, + call_type="aresponses", + start_time=None, + litellm_call_id="call", + function_id="fn", + ) + logging_obj.update_environment_variables( + litellm_params={"custom_llm_provider": "openai", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + optional_params={}, + ) + logging_obj.custom_cost_per_token = {"input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6} normalize_responses_usage(response, logging_obj) hidden: Final = get_hidden_params(response) - assert hidden is not None and hidden["response_cost"] == 0.00032 + assert hidden is not None and hidden["response_cost"] == pytest.approx(20 * 2e-6 + 60 * 6e-6) assert getattr(response.usage, "cost", None) is None, "the client field is written at the proxy egress, not here" - assert logging_obj.response_cost_calculator.call_args.kwargs["result"] is response def _capture_dispatch(logged: list): diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 13663cf4dd5..08861ea7df8 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -4907,6 +4907,26 @@ def test_stream_chunk_builder_bills_a_stream_without_writing_into_its_usage(monk assert response._hidden_params["response_cost"] > 0 +def test_stream_chunk_builder_keeps_a_zero_bill_as_hidden_response_cost() -> None: + logging_obj: Final = _stream_builder_logging_obj() + logging_obj.update_environment_variables( + litellm_params={"custom_llm_provider": "openai", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + optional_params={}, + ) + logging_obj.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + chunks: Final = [ + _stream_builder_text_chunk("gpt-4o", "Hello "), + _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj + ) + + assert response is not None + assert response._hidden_params["response_cost"] == 0.0, "a free stream still tells the egress what was billed" + + def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost(): import time as time_module @@ -4949,7 +4969,7 @@ def test_stream_chunk_builder_stamped_cost_is_not_re_read_as_xai_reported_cost() assert spend_row_cost == pytest.approx(usage_cost) -def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable(): +def test_stream_chunk_builder_stamps_the_zero_bill_when_cost_is_unpriceable(): import time as time_module from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -4978,7 +4998,7 @@ def test_stream_chunk_builder_skips_stamp_when_cost_is_unpriceable(): assert response is not None assert getattr(response.usage, "cost", None) is None - assert response._hidden_params.get("response_cost") is None + assert response._hidden_params.get("response_cost") == 0.0, "the spend row reads 0, so the egress must too" def test_stream_chunk_builder_prices_alias_response_at_its_deployment_model():