From c200c2f9f8cb393d54136404b657016996622d90 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:09:17 +0000 Subject: [PATCH] fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced cost_per_token bills a finite non-negative usage.cost from any provider, split across input and output by the ratio of the token math, unless the caller priced the call itself. completion_cost drops the reported cost when the deployment has custom pricing or a base_model, so operator rates win on streams too, where stream_chunk_builder used to stamp usage.cost as the final response_cost ahead of custom pricing and margins. The xAI and Perplexity calculators lose their private copies of the same rule. Co-authored-by: AoHanBei Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 102 ++++++++++- .../streaming_chunk_builder_utils.py | 10 -- litellm/llms/perplexity/cost_calculator.py | 16 -- litellm/llms/xai/cost_calculator.py | 18 +- .../cost_calculation/cases/openai.py | 86 +++++++++ .../integration/cost_calculation/cost_map.py | 6 + .../test_streaming_handler.py | 8 +- .../test_perplexity_cost_calculator.py | 20 ++- .../unit/llms/xai/test_xai_cost_calculator.py | 10 +- tests/unit/test_cost_calculator.py | 166 ++++++++++++++++++ 10 files changed, 381 insertions(+), 61 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8331c68fe24..6ec8d00e37f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -5,7 +5,7 @@ import math import time from collections.abc import Mapping, Sequence from datetime import datetime -from functools import lru_cache +from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast @@ -22,7 +22,7 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, provider_reported_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR @@ -408,6 +408,19 @@ def _per_second_pricing_cost( return resolved_cost_per_second * seconds, 0.0 +def _billable_reported_cost(usage_object: Usage | None) -> float | None: + return _billable_header_cost(provider_reported_usage_cost(usage_object, None)) + + +def _split_reported_cost(reported_cost: float, priced: tuple[float, float] | None) -> tuple[float, float]: + """Providers report one total, so attribute it to input and output in the ratio the token math gives.""" + priced_total: Final = (priced[0] + priced[1]) if priced is not None else 0.0 + if priced is None or priced_total <= 0: + return 0.0, reported_cost + input_cost: Final = reported_cost * priced[0] / priced_total + return input_cost, reported_cost - input_cost + + def cost_per_token( model: str = "", prompt_tokens: int = 0, @@ -444,6 +457,87 @@ def cost_per_token( request_model: str | None = None, # original request model for router detection custom_model_info: OCRPricing | None = None, bedrock_model_info: ModelInfo | None = None, +) -> tuple[float, float]: + """Cost in USD for prompt and completion tokens. + + A numeric ``usage.cost`` the provider reported is the bill, split by the ratio of our own token math, + unless the caller priced the call itself with ``custom_cost_per_token`` or ``custom_cost_per_second``. + Without a reported cost the token math is the bill. + """ + priced_by_caller: Final = custom_cost_per_token is not None or custom_cost_per_second is not None + reported_cost: Final = None if priced_by_caller else _billable_reported_cost(usage_object) + rated: Final = partial( + _cost_per_token_from_rates, + model=model, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + response_time_ms=response_time_ms, + custom_llm_provider=custom_llm_provider, + region_name=region_name, + prompt_characters=prompt_characters, + completion_characters=completion_characters, + cache_creation_input_tokens=cache_creation_input_tokens, + cache_read_input_tokens=cache_read_input_tokens, + custom_cost_per_token=custom_cost_per_token, + custom_cost_per_second=custom_cost_per_second, + number_of_queries=number_of_queries, + usage_object=usage_object, + rerank_billed_units=rerank_billed_units, + call_type=call_type, + audio_transcription_file_duration=audio_transcription_file_duration, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + response=response, + request_model=request_model, + custom_model_info=custom_model_info, + bedrock_model_info=bedrock_model_info, + ) + if reported_cost is None: + return rated() + try: + return _split_reported_cost(reported_cost, rated()) + except Exception: # noqa: BLE001 # an unmapped model has no rates to split by, the reported total still bills + verbose_logger.debug("No rates for model=%s, billing the provider reported cost %s", model, reported_cost) + return _split_reported_cost(reported_cost, None) + + +def _cost_per_token_from_rates( + model: str = "", + prompt_tokens: int = 0, + completion_tokens: int = 0, + response_time_ms: float | None = 0.0, + custom_llm_provider: str | None = None, + region_name=None, + ### CHARACTER PRICING ### + prompt_characters: int | None = None, + completion_characters: int | None = None, + ### PROMPT CACHING PRICING ### - used for anthropic + cache_creation_input_tokens: int | None = 0, + cache_read_input_tokens: int | None = 0, + ### CUSTOM PRICING ### + custom_cost_per_token: CostPerToken | None = None, + custom_cost_per_second: float | None = None, + ### NUMBER OF QUERIES ### + number_of_queries: int | None = None, + ### USAGE OBJECT ### + usage_object: Usage | None = None, # just read the usage object if provided + ### BILLED UNITS ### + rerank_billed_units: RerankBilledUnits | None = None, + ### CALL TYPE ### + call_type: CallTypesLiteral = "completion", + audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds + ### SERVICE TIER ### + service_tier: str | None = None, # for OpenAI service tier pricing + ### DATA RESIDENCY ### + data_residency: str | None = None, # for OpenAI regional-processing uplift (e.g. "eu", "us") + ### VERTEX LOCATION ### + vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global") + response: object | None = None, + ### REQUEST MODEL ### + request_model: str | None = None, # original request model for router detection + custom_model_info: OCRPricing | None = None, + bedrock_model_info: ModelInfo | None = None, ) -> tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -1610,7 +1704,9 @@ def completion_cost( audio_transcription_file_duration: float = 0.0 provider_usage_object: Final = get_usage_object(completion_response=completion_response) cost_per_token_usage_object: Final[Usage | None] = ( - _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object + _without_provider_stated_cost(provider_usage_object) + if custom_pricing or base_model is not None + else provider_usage_object ) rerank_billed_units: RerankBilledUnits | None = None diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 68cdc204af4..4cdb9004619 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1350,9 +1350,6 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) - - def _logged_provider(logging_obj: Optional["Logging"]) -> object: if logging_obj is None: return None @@ -1360,10 +1357,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object: return provider -def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: - return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS - - def _stream_builder_response_cost( response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] ) -> float | None: @@ -1374,9 +1367,6 @@ def _stream_builder_response_cost( for chunk in reversed(chunks) ) return next((cost for cost in reported if cost is not None), None) - usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) - if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): - return float(usage_cost) if logging_obj is not None: return None provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 2d762adf958..a6825eacfad 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -24,22 +24,6 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - ## USE PRE-CALCULATED COST FROM PERPLEXITY IF AVAILABLE - ## Perplexity returns accurate cost in usage.cost.total_cost including request fees. - ## By the time it reaches here, ResponseAPIUsage.parse_cost has already flattened - ## that dict down to a float, so both shapes must be accepted. - cost_info: Final = getattr(usage, "cost", None) - total_cost: float | None = None - if isinstance(cost_info, dict): - total_cost = cost_info.get("total_cost") - elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool): - total_cost = float(cost_info) - if total_cost is not None: - # Return total cost as completion_cost (prompt_cost=0) since Perplexity - # doesn't break down by input/output in their cost object - return (0.0, float(total_cost)) - - ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it ## GET MODEL INFO model_info: Final = get_model_info(model=model, custom_llm_provider="perplexity") diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index aec535c2086..b62ed39c4fa 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -1,8 +1,8 @@ """ Helper util for handling XAI-specific cost calculation -- Prefers the cost xAI reports on the response over recomputing it locally - Uses the generic cost calculator which already handles tiered pricing correctly - Handles XAI-specific reasoning token billing (billed as part of completion tokens) +- Skips the web search surcharge when xAI reported a total that already covers it """ import math @@ -49,19 +49,6 @@ def _cost_reported_by_xai(usage: "Usage") -> float | None: return float(reported_cost) -def _split_reported_cost(model: str, usage: Usage, reported_cost: float) -> tuple[float, float]: - """xAI reports one total, so attribute it to input and output in the ratio our own token math gives.""" - try: - priced_input, priced_output = _token_math_cost_per_token(model, usage) - except Exception: # noqa: BLE001 # generic_cost_per_token raises bare Exception for unmapped models - return 0.0, reported_cost - priced_total: Final = priced_input + priced_output - if priced_total <= 0: - return 0.0, reported_cost - input_cost: Final = reported_cost * priced_input / priced_total - return input_cost, reported_cost - input_cost - - def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: """ Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens. @@ -74,9 +61,6 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - reported_cost: Final = _cost_reported_by_xai(usage) - if reported_cost is not None: - return _split_reported_cost(model, usage, reported_cost) return _token_math_cost_per_token(model, usage) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 53fd5c1cec2..5e18b8dec45 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -11878,7 +11878,89 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS } ) + +# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill, +# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed +# replies alike. A deployment the operator priced keeps the operator's rates on both. +_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected( + spend=0.037, input_cost=0.023125, output_cost=0.013875, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06) +_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected( + spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_CHUNK: Final = { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1, + "model": "gateway-reported-cost-fixture", +} + +GATEWAY_REPORTED_COST: Final = CostTrackingTestCase( + name="gateway-reported-cost", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False}, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1, + "model": "gateway-reported-cost-fixture", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": _GATEWAY_REPORTED_USAGE, + }, + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase( + name="gateway-reported-cost-stream", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "reported cost"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE}, + done=True, + ), + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( + GATEWAY_REPORTED_COST, + GATEWAY_REPORTED_COST_STREAM, + GATEWAY_REPORTED_COST_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12120,6 +12202,10 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( + StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index cd979528fe6..f620b88a86b 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_above_128k_tokens": 2.2e-07, "output_cost_per_token_above_128k_tokens": 8.8e-07, }, + "gateway-reported-cost-fixture": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + }, "openrouter/anthropic/claude-sonnet-5": { "litellm_provider": "openrouter", "mode": "chat", diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index c13cea51183..9e8cd46e5ea 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2401,14 +2401,14 @@ def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator(): ) == pytest.approx(0.00503) -def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): +def test_openai_compatible_streaming_cost_is_the_reported_cost(): import litellm from litellm.cost_calculator import ( get_response_cost_from_hidden_params, response_cost_calculator, ) - model = "openai/streams-cost-in-nanodollars" + model = "openai/streams-with-a-reported-cost" litellm.register_model( { model: { @@ -2423,7 +2423,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): id="chatcmpl-openai-compatible", model=model, choices=[], - usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000), + usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.037), ) CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai") @@ -2435,7 +2435,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): custom_llm_provider="openai", call_type="completion", optional_params={}, - ) == pytest.approx(2e-5) + ) == pytest.approx(0.037) def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch): diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 83c71479311..8812b227b8c 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -15,6 +15,7 @@ import pytest # Add the project root to Python path import litellm +from litellm.cost_calculator import cost_per_token as billed_cost_per_token from litellm.llms.perplexity.cost_calculator import ( cost_per_token as perplexity_cost_per_token, ) @@ -109,13 +110,13 @@ class TestPerplexityCostCalculator: "total_cost": 0.008, } - prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage + ) - # When Perplexity provides total_cost, we use it directly - # prompt_cost should be 0, completion_cost should be total_cost - assert prompt_cost == 0.0 - assert completion_cost == 0.008 - assert prompt_cost + completion_cost == 0.008 + # When Perplexity provides total_cost, that total is the bill + assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10) + assert prompt_cost > 0.0 and completion_cost > 0.0 def test_uses_perplexity_provided_cost_when_normalized_to_float(self): """ @@ -135,10 +136,11 @@ class TestPerplexityCostCalculator: usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) usage.cost = 0.008 - prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model="perplexity/sonar-pro", custom_llm_provider="perplexity", usage_object=usage + ) - assert prompt_cost == 0.0 - assert completion_cost == 0.008 + assert math.isclose(prompt_cost + completion_cost, 0.008, rel_tol=1e-10) OFF_PEAK_MODEL = "sonar-off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" diff --git a/tests/unit/llms/xai/test_xai_cost_calculator.py b/tests/unit/llms/xai/test_xai_cost_calculator.py index 5219dba42ba..c1d2ee5f63d 100644 --- a/tests/unit/llms/xai/test_xai_cost_calculator.py +++ b/tests/unit/llms/xai/test_xai_cost_calculator.py @@ -6,6 +6,7 @@ import math import os import litellm +from litellm.cost_calculator import cost_per_token as billed_cost_per_token from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -115,7 +116,9 @@ class TestXAICostCalculator: model="grok-4.7", usage=Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) ) - prompt_cost, completion_cost = cost_per_token(model="grok-4.7", usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model="xai/grok-4.7", custom_llm_provider="xai", usage_object=usage + ) assert math.isclose(prompt_cost + completion_cost, 0.0037756, rel_tol=1e-10) assert math.isclose(prompt_cost / completion_cost, priced_prompt / priced_completion, rel_tol=1e-10) @@ -165,7 +168,10 @@ class TestXAICostCalculator: """A reported zero is a real answer, not a missing value.""" usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300, cost=0.0) - assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0) + assert billed_cost_per_token(model="xai/grok-4-latest", custom_llm_provider="xai", usage_object=usage) == ( + 0.0, + 0.0, + ) def test_custom_pricing_beats_the_reported_cost(self): response = ModelResponse( diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 27b25c1f98e..8702a79807e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10522,3 +10522,169 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry is not None assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) + + +_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037} +_REPORTED_COST_FIXTURE_MODEL: Final = "openai/reported-cost-fixture" +_REPORTED_COST_FIXTURE_RATES: Final = { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 3e-6, +} + + +def _reported_cost_logging(stream: bool, litellm_params: Mapping[str, object]) -> Logging: + logging_obj: Final = Logging( + model=_REPORTED_COST_FIXTURE_MODEL, + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="reported-cost", + function_id="reported-cost", + ) + logging_obj.update_from_kwargs( + kwargs={}, optional_params={}, litellm_params={"custom_llm_provider": "openai", **litellm_params} + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + return logging_obj + + +def _reported_cost_stream_chunk(content: str | None, finish_reason: str | None, usage: Usage | None) -> object: + return litellm.ModelResponseStream( + id="chatcmpl-reported", + model="reported-cost-fixture", + created=1, + object="chat.completion.chunk", + choices=[litellm.types.utils.StreamingChoices(index=0, delta=litellm.types.utils.Delta(content=content), finish_reason=finish_reason)], + usage=usage, + ) + + +def _bill_reported_cost(stream: bool, litellm_params: Mapping[str, object], hidden_params: Mapping[str, object]) -> float | None: + """The spend the proxy logs for one upstream reply carrying usage.cost, streamed or not.""" + logging_obj: Final = _reported_cost_logging(stream, litellm_params) + if stream: + response: Final[object] = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, Usage(**_REPORTED_USAGE)), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + else: + response = ModelResponse( + id="chatcmpl-reported", + model="reported-cost-fixture", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=Usage(**_REPORTED_USAGE), + ) + assert isinstance(response, ModelResponse) + response._hidden_params.update({"custom_llm_provider": "openai", **hidden_params}) + return logging_obj.response_cost_calculator(result=response) + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("mapped", [False, True]) +def test_provider_reported_usage_cost_is_billed_on_stream_and_non_stream( + monkeypatch: pytest.MonkeyPatch, stream: bool, mapped: bool +) -> None: + """Regression for #47533: a plain openai-compatible deployment billed usage.cost on streams and $0 on non-stream.""" + if mapped: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + else: + monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False) + + assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_deployment_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: + """Streams used to bill usage.cost even when the operator priced the deployment; both paths bill the operator's rates.""" + deployment_id: Final = "reported-cost-deployment-priced-by-the-operator" + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 2e-6, "output_cost_per_token": 6e-6}, + ) + litellm_params: Final = { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 6e-6, + "model_info": {"id": deployment_id}, + } + + assert _bill_reported_cost(stream, litellm_params, {"model_id": deployment_id}) == pytest.approx( + 100 * 2e-6 + 20 * 6e-6 + ) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_base_model_pricing_beats_the_provider_reported_usage_cost(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: + base_model: Final = "openai/reported-cost-base-model-fixture" + monkeypatch.setitem( + litellm.model_cost, + base_model, + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 5e-6, "output_cost_per_token": 7e-6}, + ) + + assert _bill_reported_cost(stream, {"base_model": base_model}, {}) == pytest.approx(100 * 5e-6 + 20 * 7e-6) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_provider_reported_usage_cost_takes_discount_then_margin(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.2}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"percentage": 0.5, "fixed_amount": 0.001}}) + + assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037 * 0.8 * 1.5 + 0.001) + + +def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + usage: Final = Usage(**_REPORTED_USAGE) + + prompt_cost, completion_cost = cost_per_token( + model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=usage + ) + + priced_prompt: Final = 100 * 1e-6 + priced_completion: Final = 20 * 3e-6 + assert prompt_cost + completion_cost == pytest.approx(0.037) + assert prompt_cost == pytest.approx(0.037 * priced_prompt / (priced_prompt + priced_completion)) + + +def test_provider_reported_usage_cost_without_rates_is_all_output(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, raising=False) + + assert cost_per_token( + model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=Usage(**_REPORTED_USAGE) + ) == (0.0, 0.037) + + +@pytest.mark.parametrize("reported", [-0.037, float("nan"), float("inf"), True]) +def test_unusable_provider_reported_usage_cost_falls_back_to_the_token_math( + monkeypatch: pytest.MonkeyPatch, reported: object +) -> None: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + usage: Final = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120) + setattr(usage, "cost", reported) + + assert cost_per_token(model=_REPORTED_COST_FIXTURE_MODEL, custom_llm_provider="openai", usage_object=usage) == ( + pytest.approx(100 * 1e-6), + pytest.approx(20 * 3e-6), + ) + + +def test_caller_supplied_custom_cost_per_token_beats_the_provider_reported_usage_cost() -> None: + assert cost_per_token( + model=_REPORTED_COST_FIXTURE_MODEL, + custom_llm_provider="openai", + prompt_tokens=100, + completion_tokens=20, + usage_object=Usage(**_REPORTED_USAGE), + custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}, + ) == (pytest.approx(0.1), pytest.approx(0.04))