diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index cacf130aeab..2eef729296c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -25,7 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping, provider_reported_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, usage_cost_stamped_by_litellm from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -1705,7 +1705,7 @@ def completion_cost( provider_usage_object: Final = get_usage_object(completion_response=completion_response) cost_per_token_usage_object: Final[Usage | None] = ( _without_provider_stated_cost(provider_usage_object) - if custom_pricing or base_model is not None + if custom_pricing or base_model is not None or usage_cost_stamped_by_litellm(completion_response) else provider_usage_object ) rerank_billed_units: RerankBilledUnits | None = None diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index a74337d2549..30082f1af7f 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -134,3 +134,17 @@ def get_or_create_hidden_params(obj: object) -> MutableMapping[str, object]: if hidden_params_mapping is not None: return hidden_params_mapping raise TypeError(f"unsupported hidden params storage: {type(hidden_params).__name__}") + + +USAGE_COST_STAMPED_BY_LITELLM: Final = "usage_cost_stamped_by_litellm" + + +def stamp_billed_usage_cost(obj: object, usage: object, cost: float) -> None: + """usage.cost becomes what litellm billed, marked so the calculator never re-bills it as a provider's report.""" + setattr(usage, "cost", cost) + set_hidden_param(obj, USAGE_COST_STAMPED_BY_LITELLM, True) + + +def usage_cost_stamped_by_litellm(obj: object) -> bool: + hidden_params: Final = get_hidden_params(obj) + return hidden_params is not None and hidden_params.get(USAGE_COST_STAMPED_BY_LITELLM) is True diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 3a5a2cbe914..024ddf0bd91 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,7 +13,7 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params, stamp_billed_usage_cost from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict ) @@ -1416,7 +1416,7 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o computed_cost: Final = logging_obj.response_cost_calculator(result=response) if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: return - setattr(usage, "cost", computed_cost) + stamp_billed_usage_cost(response, usage, computed_cost) _set_hidden_response_cost(response, float(computed_cost)) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 50ab3e4f75a..0d840d136d0 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,6 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base +from litellm.litellm_core_utils.hidden_params import stamp_billed_usage_cost from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -1757,7 +1758,7 @@ def stamp_responses_usage_cost( except Exception: return if isinstance(cost, (int, float)) and cost > 0: - setattr(usage_obj, "cost", cost) + stamp_billed_usage_cost(response_obj, usage_obj, cost) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 8812b227b8c..cd25b7a9c3d 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -228,7 +228,9 @@ class TestPerplexityCostCalculator: usage = Usage(prompt_tokens=1000, completion_tokens=200, total_tokens=1200) usage.cost = {"total_cost": 0.00501} - prompt_cost, completion_cost = perplexity_cost_per_token(model=self.OFF_PEAK_MODEL, usage=usage) + prompt_cost, completion_cost = billed_cost_per_token( + model=self.OFF_PEAK_MODEL, custom_llm_provider="perplexity", usage_object=usage + ) - assert prompt_cost == 0.0 - assert completion_cost == 0.00501 + assert math.isclose(prompt_cost + completion_cost, 0.00501, rel_tol=1e-10) + assert prompt_cost > 0.0 and completion_cost > 0.0 diff --git a/tests/unit/llms/xai/responses/test_xai_responses_transformation.py b/tests/unit/llms/xai/responses/test_xai_responses_transformation.py index 188ca453364..95ff98ac024 100644 --- a/tests/unit/llms/xai/responses/test_xai_responses_transformation.py +++ b/tests/unit/llms/xai/responses/test_xai_responses_transformation.py @@ -11,7 +11,7 @@ from unittest.mock import MagicMock, Mock import httpx -from litellm.llms.xai.cost_calculator import cost_per_token +from litellm.cost_calculator import cost_per_token from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( @@ -498,7 +498,7 @@ class TestXAIResponsesReportedCost: assert usage.cost == 0.0037756 chat_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage) - assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=chat_usage) == (0.0, 0.0037756) def test_streamed_reported_cost_reaches_the_cost_calculator(self): event = XAIResponsesAPIConfig().transform_streaming_response( @@ -520,7 +520,7 @@ class TestXAIResponsesReportedCost: assert isinstance(event, ResponseCompletedEvent) chat_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(event.response.usage) - assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=chat_usage) == (0.0, 0.0037756) def test_usage_without_a_reported_cost_is_left_alone(self): usage = self._transformed_usage({"input_tokens": 100, "output_tokens": 200, "total_tokens": 300}) diff --git a/tests/unit/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py index 69e620dbc8a..3aa35759662 100644 --- a/tests/unit/llms/xai/test_xai_chat_transformation.py +++ b/tests/unit/llms/xai/test_xai_chat_transformation.py @@ -13,7 +13,7 @@ from litellm.llms.xai.chat.transformation import ( XAIChatCompletionStreamingHandler, XAIChatConfig, ) -from litellm.llms.xai.cost_calculator import cost_per_token +from litellm.cost_calculator import cost_per_token from litellm.types.utils import ( CompletionTokensDetailsWrapper, ModelResponse, @@ -433,7 +433,7 @@ class TestXAIReportedCost: ) assert usage.cost == 0.0037756 - assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0037756) + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=usage) == (0.0, 0.0037756) def test_usage_without_a_reported_cost_is_left_alone(self): usage = self._transformed_usage({"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}) @@ -481,7 +481,7 @@ class TestXAIReportedCost: assembled = litellm.stream_chunk_builder(chunks=[parsed]) assert assembled.usage.cost == 0.0037756 - assert cost_per_token(model="grok-4-latest", usage=assembled.usage) == ( + assert cost_per_token(model="grok-4-latest", custom_llm_provider="xai", usage_object=assembled.usage) == ( 0.0, 0.0037756, ) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 8702a79807e..81b00f3225d 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10641,6 +10641,31 @@ def test_provider_reported_usage_cost_takes_discount_then_margin(monkeypatch: py assert _bill_reported_cost(stream, {}, {}) == pytest.approx(0.037 * 0.8 * 1.5 + 0.001) +def test_a_usage_cost_litellm_stamped_on_a_stream_is_not_billed_again_as_a_provider_report( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem(litellm.model_cost, _REPORTED_COST_FIXTURE_MODEL, _REPORTED_COST_FIXTURE_RATES) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.002}}) + logging_obj: Final = _reported_cost_logging(True, {}) + token_math_plus_margin: Final = 100 * 1e-6 + 20 * 3e-6 + 0.002 + + response: Final[object] = litellm.stream_chunk_builder( + [ + _reported_cost_stream_chunk("hi", None, None), + _reported_cost_stream_chunk(None, "stop", None), + _reported_cost_stream_chunk(None, None, Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)), + ], + messages=[{"role": "user", "content": "hi"}], + logging_obj=logging_obj, + ) + + assert isinstance(response, ModelResponse) + assert response.usage is not None and response.usage.cost == pytest.approx(token_math_plus_margin) + assert logging_obj.response_cost_calculator(result=response, ignore_stamped_cost=True) == pytest.approx( + token_math_plus_margin + ) + + def test_provider_reported_usage_cost_splits_input_and_output_by_the_token_math_ratio( monkeypatch: pytest.MonkeyPatch, ) -> None: