diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b83e9b395a8..cb7da32f857 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -4,6 +4,7 @@ import logging import time from collections.abc import Mapping, Sequence from functools import lru_cache +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast from httpx import Response @@ -1164,6 +1165,12 @@ def _store_cost_breakdown_in_logging_obj( # Don't fail the main cost calculation if breakdown storage fails +def _without_provider_stated_cost(usage: Usage | None) -> Usage | None: + if usage is None or getattr(usage, "cost", None) is None: + return usage + return usage.model_copy(update=MappingProxyType({"cost": None})) + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -1243,7 +1250,10 @@ def completion_cost( cache_creation_input_tokens: int | None = None cache_read_input_tokens: int | None = None audio_transcription_file_duration: float = 0.0 - cost_per_token_usage_object: Final[Usage | None] = _get_usage_object(completion_response=completion_response) + provider_usage_object: Final = _get_usage_object(completion_response=completion_response) + cost_per_token_usage_object: Final[Usage | None] = ( + _without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object + ) rerank_billed_units: RerankBilledUnits | None = None # Extract service_tier from optional_params if not provided directly diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 480b1921c18..355faa41e68 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -54,6 +54,7 @@ FUNCTION_CALL_ATTRIBUTE: Final = "function_call" _SYNC_ITER_EXHAUSTED: Final = object() _GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__) +_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.OPENROUTER.value}) def _next_sync_or_exhausted(it: Any) -> object: @@ -1886,8 +1887,8 @@ class CustomStreamWrapper: @staticmethod def _resolve_provider_reported_cost(usage_cost: object) -> float | None: """ - Providers report usage.cost either as a number or, for Perplexity, as a - breakdown object whose total lives under ``total_cost``. + Providers report usage.cost either as a number or as a breakdown object + whose total lives under ``total_cost``. """ if isinstance(usage_cost, bool): return None @@ -1900,12 +1901,10 @@ class CustomStreamWrapper: @staticmethod def _propagate_usage_cost_to_hidden_params( response: "ModelResponse", + custom_llm_provider: str | None, ) -> None: - """ - If the assembled response carries a provider-reported cost on - usage.cost, copy it into _hidden_params so litellm's cost - calculator uses it instead of a token-based estimate. - """ + if custom_llm_provider not in _USAGE_COST_HEADER_PROVIDERS: + return _usage: Final[Usage | None] = getattr(response, "usage", None) _cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None)) if _cost is not None: @@ -2020,7 +2019,7 @@ class CustomStreamWrapper: response = self.model_response_creator() if complete_streaming_response is not None: - self._propagate_usage_cost_to_hidden_params(complete_streaming_response) + self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) setattr( response, @@ -2270,7 +2269,7 @@ class CustomStreamWrapper: response: Final = self.model_response_creator() if complete_streaming_response is not None: - self._propagate_usage_cost_to_hidden_params(complete_streaming_response) + self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) setattr( response, diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index ae5849812bf..09af8544b28 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -1,4 +1,5 @@ from collections.abc import AsyncIterator, Iterator, Mapping +from types import MappingProxyType from typing import Any, Final import httpx @@ -11,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( filter_value_from_dict, strip_name_from_messages, ) -from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd from litellm.llms.xai.cost_calculator import ( apply_server_side_tool_usage_details_to_usage, ) @@ -30,6 +31,13 @@ from ...openai.chat.gpt_transformation import ( ) +def _usage_restated_from_xai_ticks(usage: Usage | None) -> Usage | None: + reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None)) + if usage is None or reported_cost is None: + return None + return usage.model_copy(update=MappingProxyType({"cost": reported_cost})) + + class XAIChatConfig(OpenAIGPTConfig): @property def custom_llm_provider(self) -> str | None: @@ -283,6 +291,9 @@ class XAIChatConfig(OpenAIGPTConfig): self._fold_reasoning_tokens_into_completion(response) self._normalize_openai_compatible_usage_totals(getattr(response, "usage", None)) + restated_usage: Final = _usage_restated_from_xai_ticks(getattr(response, "usage", None)) + if restated_usage is not None: + response.usage = restated_usage return response @staticmethod @@ -411,4 +422,8 @@ class XAIChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler): XAIChatConfig._fold_reasoning_tokens_into_completion(chunk["usage"]) XAIChatConfig._normalize_openai_compatible_usage_totals(chunk["usage"]) - return super().chunk_parser(chunk) + parsed_chunk: Final = super().chunk_parser(chunk) + restated_usage: Final = _usage_restated_from_xai_ticks(getattr(parsed_chunk, "usage", None)) + if restated_usage is not None: + parsed_chunk.usage = restated_usage + return parsed_chunk diff --git a/litellm/llms/xai/common_utils.py b/litellm/llms/xai/common_utils.py index f122098332e..cf76a851a86 100644 --- a/litellm/llms/xai/common_utils.py +++ b/litellm/llms/xai/common_utils.py @@ -8,6 +8,17 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ProviderSpecificModelInfo +USD_TICKS_PER_DOLLAR: Final = 10_000_000_000 + + +def xai_reported_cost_in_usd(cost_in_usd_ticks: object) -> float | None: + """xAI bills in ticks of a dollar: https://docs.x.ai/developers/cost-tracking""" + if not isinstance(cost_in_usd_ticks, int) or isinstance(cost_in_usd_ticks, bool): + return None + if cost_in_usd_ticks < 0: + return None + return cost_in_usd_ticks / USD_TICKS_PER_DOLLAR + class XAIModelInfo(BaseLLMModelInfo): def get_provider_info( diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index dd77b8d5d09..164568451d4 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -1,9 +1,11 @@ """ Helper util for handling XAI-specific cost calculation +- Prefers the cost xAI reports on the response over recomputing it locally - Uses the generic cost calculator which already handles tiered pricing correctly - Handles XAI-specific reasoning token billing (billed as part of completion tokens) """ +import math from collections.abc import Mapping from typing import TYPE_CHECKING, Final @@ -36,6 +38,17 @@ def apply_server_side_tool_usage_details_to_usage(usage: Usage, details: Mapping usage.prompt_tokens_details = prompt_tokens_details # rebind-ok: write details onto caller usage +def _cost_reported_by_xai(usage: "Usage") -> float | None: + reported_cost: Final[object] = getattr(usage, "cost", None) + if not isinstance(reported_cost, (int, float)) or isinstance(reported_cost, bool): + return None + if not math.isfinite(reported_cost): + return None + if reported_cost < 0: + return None + return float(reported_cost) + + def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: """ Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens. @@ -48,6 +61,10 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ + reported_cost: Final = _cost_reported_by_xai(usage) + if reported_cost is not None: + return 0.0, reported_cost + # XAI-specific completion cost: completion is billed as visible + reasoning # tokens. Detect when the transformation layer already folded them so we # don't double-count; fall back to raw xAI shape for callers that bypass @@ -112,6 +129,9 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa Per-call rate comes from model_info.search_context_cost_per_query when set, otherwise the default xAI tools rate ($5 / 1k calls). """ + if _cost_reported_by_xai(usage) is not None: + return 0.0 + details: Final = getattr(usage, "server_side_tool_usage_details", None) if not isinstance(details, Mapping): return 0.0 diff --git a/litellm/llms/xai/responses/transformation.py b/litellm/llms/xai/responses/transformation.py index d79e7d4c146..acf03d88911 100644 --- a/litellm/llms/xai/responses/transformation.py +++ b/litellm/llms/xai/responses/transformation.py @@ -1,17 +1,44 @@ -from typing import Any, Final +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final + +import httpx import litellm from litellm._logging import verbose_logger from litellm.constants import XAI_API_BASE from litellm.exceptions import AuthenticationError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponseFailedEvent, + ResponseIncompleteEvent, + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, + ResponsesAPIStreamingResponse, +) from litellm.types.llms.xai import XAIWebSearchTool, XAIXSearchTool from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as _LiteLLMLoggingObj, + ) + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +def _usage_restated_from_xai_ticks(usage: ResponseAPIUsage | None) -> ResponseAPIUsage | None: + reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None)) + if usage is None or reported_cost is None: + return None + return usage.model_copy(update=MappingProxyType({"cost": reported_cost})) + class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -250,6 +277,41 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return f"{api_base}/responses" + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + response: Final = super().transform_response_api_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + ) + + restated_usage: Final = _usage_restated_from_xai_ticks(response.usage) + if restated_usage is not None: + response.usage = restated_usage + return response + + def transform_streaming_response( + self, + model: str, + parsed_chunk: dict, # mutable-ok: overrides the base class signature + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIStreamingResponse: + event: Final = super().transform_streaming_response( + model=model, + parsed_chunk=parsed_chunk, + logging_obj=logging_obj, + ) + if not isinstance(event, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)): + return event + restated_usage: Final = _usage_restated_from_xai_ticks(event.response.usage) + if restated_usage is not None: + event.response.usage = restated_usage + return event + def supports_native_websocket(self) -> bool: """XAI does not support native WebSocket for Responses API""" return False diff --git a/litellm/main.py b/litellm/main.py index 0128e4defe5..0bca4a7350e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8595,9 +8595,19 @@ def stream_chunk_builder_text_completion(chunks: list, messages: list | None = N return TextCompletionResponse(**response) +_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) + + +def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: + if logging_obj is None: + return False + provider: Final[object] = logging_obj.model_call_details.get("custom_llm_provider") + return provider in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS + + def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None: usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) - if isinstance(usage_cost, (int, float)): + if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): return float(usage_cost) if logging_obj is not None: return None diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 4c99bce2b0f..4b2a3f9247c 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -21,6 +21,7 @@ from litellm.litellm_core_utils.streaming_handler import ( from litellm.types.utils import ( CompletionTokensDetailsWrapper, Delta, + ModelResponse, ModelResponseStream, PromptTokensDetailsWrapper, StandardLoggingPayload, @@ -1750,7 +1751,7 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params(): assert complete_response.usage.cost == 0.00025 # Use the real propagation method from CustomStreamWrapper - CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response) + CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openrouter") assert "additional_headers" in complete_response._hidden_params assert ( @@ -1769,14 +1770,12 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params(): assert provider_cost == 0.00025 -def test_perplexity_streaming_dict_cost_propagates_to_hidden_params(): - """ - Regression: Perplexity reports usage.cost as a breakdown object, which used to - blow up the end of the stream with - `float() argument must be a string or a real number, not 'dict'`. - """ +def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator(): import litellm - from litellm.cost_calculator import get_response_cost_from_hidden_params + from litellm.cost_calculator import ( + get_response_cost_from_hidden_params, + response_cost_calculator, + ) chunks = [ ModelResponseStream( @@ -1828,13 +1827,81 @@ def test_perplexity_streaming_dict_cost_propagates_to_hidden_params(): assert complete_response is not None - CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response) + CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "perplexity") - assert ( - get_response_cost_from_hidden_params(complete_response._hidden_params) - == 0.00503 + assert get_response_cost_from_hidden_params(complete_response._hidden_params) is None + assert response_cost_calculator( + response_object=complete_response, + model="perplexity/sonar", + custom_llm_provider="perplexity", + call_type="completion", + optional_params={}, + ) == pytest.approx(0.00503) + + +def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): + import litellm + from litellm.cost_calculator import ( + get_response_cost_from_hidden_params, + response_cost_calculator, ) + model = "openai/streams-cost-in-nanodollars" + litellm.register_model( + { + model: { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "litellm_provider": "openai", + "mode": "chat", + } + } + ) + complete_response = ModelResponse( + id="chatcmpl-openai-compatible", + model=model, + choices=[], + usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000), + ) + + CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai") + + assert get_response_cost_from_hidden_params(complete_response._hidden_params) is None + assert response_cost_calculator( + response_object=complete_response, + model=model, + custom_llm_provider="openai", + call_type="completion", + optional_params={}, + ) == pytest.approx(2e-5) + + +def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch): + import litellm + from litellm.cost_calculator import ( + get_response_cost_from_hidden_params, + response_cost_calculator, + ) + + complete_response = ModelResponse( + id="chatcmpl-xai", + model="grok-4-latest", + choices=[], + usage=Usage(completion_tokens=353, prompt_tokens=198, total_tokens=551, cost=0.0009956), + ) + + CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "xai") + + assert get_response_cost_from_hidden_params(complete_response._hidden_params) is None + monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5}) + assert response_cost_calculator( + response_object=complete_response, + model="xai/grok-4-latest", + custom_llm_provider="xai", + call_type="completion", + optional_params={}, + ) == pytest.approx(0.0009956 * 1.5) + def test_provider_reported_cost_ignores_unusable_shapes(): assert CustomStreamWrapper._resolve_provider_reported_cost(None) is None diff --git a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py index befd4c5ffbd..4cff5c76b9e 100644 --- a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py +++ b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py @@ -7,12 +7,13 @@ transformations for the Responses API. Source: litellm/llms/xai/responses/transformation.py """ -from unittest.mock import MagicMock - +from unittest.mock import MagicMock, Mock +import httpx import pytest import litellm +from litellm.llms.xai.cost_calculator import cost_per_token from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( @@ -400,3 +401,94 @@ class TestXAIResponsesWebSearchBilling: bridged = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(event.response.usage) assert getattr(bridged, "server_side_tool_usage_details") == self._TOOL_DETAILS + + +class TestXAIResponsesReportedCost: + """xAI reports what it charged; the transformation moves it to where litellm bills from. + + ``ResponseAPILoggingUtils`` copies ``usage.cost`` onto the chat Usage that cost + tracking prices, so restating ``cost_in_usd_ticks`` there is what makes /v1/responses + bill the reported figure. At 10^10 ticks to the dollar, 37756000 ticks is $0.0037756. + """ + + @staticmethod + def _response_body(usage: dict) -> dict: + return { + "id": "resp_xai", + "object": "response", + "created_at": 0, + "model": "grok-4-latest", + "status": "completed", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": usage, + } + + def _transformed_usage(self, usage: dict) -> ResponseAPIUsage | None: + raw_response = httpx.Response(status_code=200, json=self._response_body(usage)) + + response = XAIResponsesAPIConfig().transform_response_api_response( + model="grok-4-latest", + raw_response=raw_response, + logging_obj=Mock(), + ) + return response.usage + + def test_reported_cost_reaches_the_cost_calculator(self): + usage = self._transformed_usage( + { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": 37756000, + } + ) + + assert usage.cost == 0.0037756 + + chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) + + def test_streamed_reported_cost_reaches_the_cost_calculator(self): + event = XAIResponsesAPIConfig().transform_streaming_response( + model="grok-4-latest", + parsed_chunk={ + "type": "response.completed", + "sequence_number": 7, + "response": self._response_body( + { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": 37756000, + } + ), + }, + logging_obj=Mock(), + ) + + assert isinstance(event, ResponseCompletedEvent) + chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(event.response.usage) + assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) + + def test_usage_without_a_reported_cost_is_left_alone(self): + usage = self._transformed_usage( + {"input_tokens": 100, "output_tokens": 200, "total_tokens": 300} + ) + + assert usage.cost is None + + def test_negative_reported_cost_is_not_carried(self): + """A caller who can set api_base must not be able to report negative spend.""" + usage = self._transformed_usage( + { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": -37756000, + } + ) + + assert usage.cost is None diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py index e5e853ec82f..c67dca11a56 100644 --- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py +++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py @@ -1,9 +1,14 @@ +from unittest.mock import Mock - +import httpx import pytest import litellm -from litellm.llms.xai.chat.transformation import XAIChatConfig +from litellm.llms.xai.chat.transformation import ( + XAIChatCompletionStreamingHandler, + XAIChatConfig, +) +from litellm.llms.xai.cost_calculator import cost_per_token from litellm.types.utils import ( CompletionTokensDetailsWrapper, ModelResponse, @@ -195,3 +200,113 @@ class TestXAIChatWebSearchBilling: ) assert with_search - without_search == pytest.approx(3 * 5.0 / 1000.0) + + +class TestXAIReportedCost: + """xAI reports what it charged; the transformation moves it to where litellm bills from. + + ``cost`` is the field litellm already carries a provider stated cost in, so restating + ``cost_in_usd_ticks`` there is what lets ``llms/xai/cost_calculator.py`` bill the + reported figure. At 10^10 ticks to the dollar, 37756000 ticks is $0.0037756. + """ + + @staticmethod + def _transformed_usage(usage: dict) -> Usage: + raw_response = httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-xai", + "object": "chat.completion", + "created": 0, + "model": "grok-4-latest", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": usage, + }, + ) + + response = XAIChatConfig().transform_response( + model="grok-4-latest", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=Mock(), + request_data={}, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + return response.usage + + def test_reported_cost_reaches_the_cost_calculator(self): + usage = self._transformed_usage( + { + "prompt_tokens": 100, + "completion_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": 37756000, + } + ) + + assert usage.cost == 0.0037756 + assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0037756) + + def test_usage_without_a_reported_cost_is_left_alone(self): + usage = self._transformed_usage( + {"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300} + ) + + assert getattr(usage, "cost", None) is None + + def test_negative_reported_cost_is_not_carried(self): + """A caller who can set api_base must not be able to report negative spend.""" + usage = self._transformed_usage( + { + "prompt_tokens": 100, + "completion_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": -37756000, + } + ) + + assert getattr(usage, "cost", None) is None + + def test_streamed_reported_cost_survives_chunk_aggregation(self): + """Streamed spend only matches if the conversion happens on the chunk. + + Chunk aggregation rebuilds usage from the fields it models plus ``cost``, so a + chunk still carrying only ``cost_in_usd_ticks`` loses the reported amount. + """ + handler = XAIChatCompletionStreamingHandler( + streaming_response=iter([]), sync_stream=True + ) + + parsed = handler.chunk_parser( + { + "id": "chatcmpl-xai", + "object": "chat.completion.chunk", + "created": 0, + "model": "grok-4-latest", + "choices": [], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": 37756000, + }, + } + ) + + assert parsed.usage.cost == 0.0037756 + + assembled = litellm.stream_chunk_builder(chunks=[parsed]) + assert assembled.usage.cost == 0.0037756 + assert cost_per_token(model="grok-4-latest", usage=assembled.usage) == ( + 0.0, + 0.0037756, + ) diff --git a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py index 92e76fd18ab..6503e956a51 100644 --- a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py +++ b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py @@ -7,7 +7,10 @@ import os import litellm from litellm.types.utils import ( + Choices, CompletionTokensDetailsWrapper, + Message, + ModelResponse, PromptTokensDetailsWrapper, Usage, ) @@ -361,6 +364,145 @@ class TestXAICostCalculator: response_object=object(), usage=usage ) + def test_reported_cost_is_preferred_over_token_math(self): + """The amount xAI reported, carried on usage.cost by the transformation, is billed. + + It lands entirely on completion cost because xAI does not split its total by + direction, the same shape the perplexity calculator returns. + """ + usage = Usage( + prompt_tokens=100, + completion_tokens=200, + total_tokens=300, + cost=0.0037756, + ) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost == 0.0 + assert math.isclose(completion_cost, 0.0037756, rel_tol=1e-10) + + def test_reported_cost_suppresses_web_search_surcharge(self): + """The reported total already covers server-side tool calls. + + Without the suppression these 3 searches would be billed a second time on + top of the total xAI already charged. + """ + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=100, + web_search_requests=3, + ), + cost=0.0037756, + ) + + assert cost_per_web_search_request(usage=usage, model_info={}) == 0.0 + + def test_web_search_surcharge_suppressed_through_the_dispatcher(self): + """The suppression has to hold on the path cost tracking actually uses. + + Legacy behaviour stays intact when xAI reports no cost. + """ + from litellm.llms import get_cost_for_web_search_request + + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) + + assert get_cost_for_web_search_request("xai", usage, {}) > 0.0 + + reported = Usage( + prompt_tokens=100, completion_tokens=50, total_tokens=150, cost=0.0037756 + ) + setattr(reported, "server_side_tool_usage_details", {"web_search_calls": 3}) + assert get_cost_for_web_search_request("xai", reported, {}) == 0.0 + + def test_no_reported_cost_falls_back_to_token_math(self): + """Absent the provider figure, nothing changes for existing callers.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + + def test_malformed_reported_cost_falls_back_to_token_math(self): + """A junk value must not fail the request, fall back to calculating.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) + setattr(usage, "cost", "not-a-number") + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + + def test_boolean_reported_cost_falls_back_to_token_math(self): + """True is an int in python and would otherwise be billed as $1.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) + setattr(usage, "cost", True) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + assert completion_cost != 1.0 + + def test_negative_reported_cost_is_rejected(self): + """A negative amount must never reach spend tracking. + + A caller who can set api_base controls the response body, so trusting a + negative figure would let them subtract from their own recorded spend and + slip past a budget. Fall back to token pricing instead, and keep charging + the web search surcharge, since no trustworthy total was reported. + """ + usage = Usage( + prompt_tokens=100, + completion_tokens=200, + total_tokens=300, + cost=-0.0037756, + ) + setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0 + + def test_non_finite_reported_cost_is_rejected(self): + """NaN compares false against every budget threshold. + + Usage stores a provider supplied cost without validating it, so a caller who + controls the response body could report NaN and leave spend >= max_budget + false for the life of the key rather than mispricing one request. The + infinities are refused alongside it. Fall back to token pricing and keep + charging the web search surcharge, since no trustworthy total was reported. + """ + for reported_cost in (float("nan"), float("inf"), float("-inf")): + usage = Usage( + prompt_tokens=100, + completion_tokens=200, + total_tokens=300, + cost=reported_cost, + ) + setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert math.isfinite(prompt_cost), reported_cost + assert math.isfinite(completion_cost), reported_cost + assert prompt_cost > 0.0, reported_cost + assert completion_cost > 0.0, reported_cost + assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0, reported_cost + + def test_zero_reported_cost_is_honoured(self): + """A reported zero is a real answer, not a missing value.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300, cost=0.0) + + assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0) + def test_grok_4_20_beta_reasoning_cost_calculation(self): """Test cost calculation for grok-4.20-beta-0309-reasoning model.""" usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) @@ -437,6 +579,48 @@ class TestXAICostCalculator: assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) + def test_custom_pricing_beats_the_reported_cost(self): + response = ModelResponse( + id="chatcmpl-xai", + model="grok-4-latest", + choices=[Choices(index=0, message=Message(role="assistant", content="x"), finish_reason="stop")], + usage=Usage(prompt_tokens=198, completion_tokens=353, total_tokens=551, cost=0.0009956), + ) + + billed = litellm.completion_cost( + completion_response=response, + model="xai/grok-4-latest", + custom_llm_provider="xai", + custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.001}, + custom_pricing=True, + ) + + assert math.isclose(billed, 0.551, rel_tol=1e-10) + + def test_deployment_custom_pricing_beats_the_reported_cost(self, monkeypatch): + deployment_id = "xai-deployment-priced-by-the-operator" + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.001, "litellm_provider": "xai", "mode": "chat"}, + ) + response = ModelResponse( + id="chatcmpl-xai", + model="grok-4-latest", + choices=[Choices(index=0, message=Message(role="assistant", content="x"), finish_reason="stop")], + usage=Usage(prompt_tokens=198, completion_tokens=353, total_tokens=551, cost=0.0009956), + ) + + billed = litellm.completion_cost( + completion_response=response, + model="xai/grok-4-latest", + custom_llm_provider="xai", + custom_pricing=True, + router_model_id=deployment_id, + ) + + assert math.isclose(billed, 0.551, rel_tol=1e-10) + class TestXAIWebSearchCostHelpers: """Focused coverage for web_search / tool-usage helpers in cost_calculator.py.""" diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 7c2b9d0be05..3c8bf142835 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -3131,9 +3131,9 @@ def test_stream_chunk_builder_prices_proxy_alias_via_model_map(): assert response._hidden_params["response_cost"] == pytest.approx(expected_cost) -def _stream_builder_logging_obj() -> LiteLLMLogging: +def _stream_builder_logging_obj(model: str = "gpt-4o", custom_llm_provider: str = "openai") -> LiteLLMLogging: logging_obj: Final = LiteLLMLogging( - model="gpt-4o", + model=model, messages=[{"role": "user", "content": "hi"}], stream=True, call_type="completion", @@ -3142,10 +3142,11 @@ def _stream_builder_logging_obj() -> LiteLLMLogging: function_id="test-function-id", ) logging_obj.update_environment_variables( - model="gpt-4o", + model=model, user=None, optional_params={}, - litellm_params={"custom_llm_provider": "openai"}, + litellm_params={"custom_llm_provider": custom_llm_provider}, + custom_llm_provider=custom_llm_provider, ) return logging_obj @@ -3237,3 +3238,24 @@ def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): assert response.usage.completion_tokens == 60 assert getattr(response.usage, "cost", None) == pytest.approx(0.000704) assert response._hidden_params["response_cost"] == pytest.approx(0.000704) + + +def test_stream_chunk_builder_leaves_xai_reported_cost_to_the_calculator(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5}) + usage_chunk: Final = _stream_builder_text_chunk("grok-4", "") + usage_chunk.usage = Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7, cost=0.42) + chunks: Final = [ + _stream_builder_text_chunk("grok-4", "Hello "), + _stream_builder_text_chunk("grok-4", "world.", finish_reason="stop"), + usage_chunk, + ] + logging_obj: Final = _stream_builder_logging_obj(model="grok-4", custom_llm_provider="xai") + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj + ) + + assert response is not None + assert getattr(response.usage, "cost", None) == pytest.approx(0.42) + assert response._hidden_params.get("response_cost") is None + assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.63)