From ddcbba09c481b7b619cd48cb144211ecaa8ee5db Mon Sep 17 00:00:00 2001 From: Acacian Date: Mon, 10 Aug 2026 22:20:32 +0900 Subject: [PATCH] fix(xai): bill from the cost xAI reports instead of recomputing it xAI states the amount it charged in usage.cost_in_usd_ticks, at 10^10 ticks to the dollar, and that figure covers tokens and every server-side tool invocation together. The xAI chat and responses transformations restate it in USD on usage.cost, the field litellm already carries a provider-stated cost in, and the xAI cost calculator bills from it the way the perplexity calculator does Routing it through usage.cost rather than a private field means the streaming chunk assembler carries it too, and no provider-neutral file has to learn about an xAI wire field Only a non-negative integer is trusted, so an endpoint a caller can point litellm at cannot report a negative amount to subtract from its own recorded spend. Absent a usable figure nothing changes: the existing token math and the $5 per 1,000 web search calls fallback both run as before The web search surcharge is suppressed once the reported total applies, since that total already covers the search calls --- litellm/llms/xai/chat/transformation.py | 31 ++++- litellm/llms/xai/common_utils.py | 24 ++++ litellm/llms/xai/cost_calculator.py | 45 ++++++- litellm/llms/xai/responses/transformation.py | 51 +++++++- .../test_xai_responses_transformation.py | 74 ++++++++++- .../llms/xai/test_xai_chat_transformation.py | 119 +++++++++++++++++- .../llms/xai/test_xai_cost_calculator.py | 113 +++++++++++++++++ 7 files changed, 444 insertions(+), 13 deletions(-) diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index ae5849812bf..1a8e54882f8 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -11,7 +11,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( filter_value_from_dict, strip_name_from_messages, ) -from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd from litellm.llms.xai.cost_calculator import ( apply_server_side_tool_usage_details_to_usage, ) @@ -30,6 +30,33 @@ from ...openai.chat.gpt_transformation import ( ) +def _adopt_cost_reported_by_xai(usage: Usage | dict[str, Any] | None) -> None: # mutable-ok: streaming dict write + """Bill what xAI charged instead of repricing the request locally. + + xAI reports the amount on ``cost_in_usd_ticks``; restate it in USD on ``cost``, + the field litellm already carries a provider stated cost in and the one + ``llms/xai/cost_calculator.py`` prices from. When xAI reported nothing usable, + ``cost`` is left alone and the request falls back to token pricing. + + Accepts a ``Usage`` (non-streaming) or a raw usage ``dict`` (streaming chunk), + matching ``_fold_reasoning_tokens_into_completion``, so both paths stay in sync. + Streaming needs the dict form because chunk aggregation rebuilds usage from the + fields it models plus ``cost``, dropping everything else xAI sent. + """ + if usage is None: + return + + if isinstance(usage, dict): + chunk_cost: Final = xai_reported_cost_in_usd(usage.get("cost_in_usd_ticks")) + if chunk_cost is not None: + usage["cost"] = chunk_cost + return + + reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None)) + if reported_cost is not None: + usage.cost = reported_cost + + class XAIChatConfig(OpenAIGPTConfig): @property def custom_llm_provider(self) -> str | None: @@ -283,6 +310,7 @@ class XAIChatConfig(OpenAIGPTConfig): self._fold_reasoning_tokens_into_completion(response) self._normalize_openai_compatible_usage_totals(getattr(response, "usage", None)) + _adopt_cost_reported_by_xai(getattr(response, "usage", None)) return response @staticmethod @@ -410,5 +438,6 @@ class XAIChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler): if "usage" in chunk and chunk["usage"] is not None: XAIChatConfig._fold_reasoning_tokens_into_completion(chunk["usage"]) XAIChatConfig._normalize_openai_compatible_usage_totals(chunk["usage"]) + _adopt_cost_reported_by_xai(chunk["usage"]) return super().chunk_parser(chunk) diff --git a/litellm/llms/xai/common_utils.py b/litellm/llms/xai/common_utils.py index f122098332e..248b440b8f0 100644 --- a/litellm/llms/xai/common_utils.py +++ b/litellm/llms/xai/common_utils.py @@ -8,6 +8,30 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ProviderSpecificModelInfo +USD_TICKS_PER_DOLLAR: Final = 10_000_000_000 + + +def xai_reported_cost_in_usd(cost_in_usd_ticks: object) -> float | None: + """ + Convert the amount xAI says it charged into USD, or None when it reported nothing usable. + + xAI states what it billed in ``usage.cost_in_usd_ticks``, at ``USD_TICKS_PER_DOLLAR`` + ticks to the dollar: https://docs.x.ai/developers/cost-tracking + That single figure covers the whole request, tokens and every server side tool + invocation together, so whoever bills from it must not add anything on top. + + The value arrives on an untyped field of a response body that a caller able to set + api_base controls, so only the documented shape is accepted: a non-negative integer, + with bool refused since it is an int subclass. Anything else yields None and the + request is priced from tokens instead, which stops such an endpoint from reporting a + negative amount to subtract from its own recorded spend. + """ + if not isinstance(cost_in_usd_ticks, int) or isinstance(cost_in_usd_ticks, bool): + return None + if cost_in_usd_ticks < 0: + return None + return cost_in_usd_ticks / USD_TICKS_PER_DOLLAR + class XAIModelInfo(BaseLLMModelInfo): def get_provider_info( diff --git a/litellm/llms/xai/cost_calculator.py b/litellm/llms/xai/cost_calculator.py index dd77b8d5d09..8b13b89619b 100644 --- a/litellm/llms/xai/cost_calculator.py +++ b/litellm/llms/xai/cost_calculator.py @@ -1,5 +1,6 @@ """ Helper util for handling XAI-specific cost calculation +- Prefers the cost xAI reports on the response over recomputing it locally - Uses the generic cost calculator which already handles tiered pricing correctly - Handles XAI-specific reasoning token billing (billed as part of completion tokens) """ @@ -36,10 +37,35 @@ def apply_server_side_tool_usage_details_to_usage(usage: Usage, details: Mapping usage.prompt_tokens_details = prompt_tokens_details # rebind-ok: write details onto caller usage +def _cost_reported_by_xai(usage: "Usage") -> float | None: + """ + Return what xAI billed for the request in USD, or None if it reported nothing usable. + + The xAI transformations restate ``usage.cost_in_usd_ticks`` as ``usage.cost``, the + field litellm already carries a provider stated cost in and the same one + ``llms/perplexity/cost_calculator.py`` bills from. That figure is the total for the + whole request, tokens and every server side tool invocation together, so nothing may + be added on top of it. + + A negative amount is refused rather than billed: a caller who can point litellm at an + api_base they control also controls the response body, and a negative cost would + subtract from their own recorded spend. Those requests are priced from tokens instead. + """ + reported_cost: Final[object] = getattr(usage, "cost", None) + if not isinstance(reported_cost, (int, float)) or isinstance(reported_cost, bool): + return None + if reported_cost < 0: + return None + return float(reported_cost) + + def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: """ - Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens. - Uses the generic cost calculator for all pricing logic, with XAI-specific reasoning token handling. + Prefers the amount xAI reported for the request, matching how the perplexity + calculator treats a provider-stated cost. That total is returned as completion + cost because xAI does not break it down by direction. Without one, falls back to + the generic cost calculator for all pricing logic, with XAI-specific reasoning + token handling. Input: - model: str, the model name without provider prefix @@ -48,6 +74,10 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]: Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ + reported_cost: Final = _cost_reported_by_xai(usage) + if reported_cost is not None: + return 0.0, reported_cost + # XAI-specific completion cost: completion is billed as visible + reasoning # tokens. Detect when the transformation layer already folded them so we # don't double-count; fall back to raw xAI shape for callers that bypass @@ -108,10 +138,15 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa """ Calculate the cost of web search requests for X.AI models. - Counts invocations from usage.server_side_tool_usage_details.web_search_calls. - Per-call rate comes from model_info.search_context_cost_per_query when set, - otherwise the default xAI tools rate ($5 / 1k calls). + When xAI reports what it billed, that figure already covers the server-side + search calls and ``cost_per_token`` has returned it, so there is nothing to add + here. Otherwise price the invocations from + usage.server_side_tool_usage_details.web_search_calls at the per-call rate + (model_info.search_context_cost_per_query when set, else the default $5 / 1k). """ + if _cost_reported_by_xai(usage) is not None: + return 0.0 + details: Final = getattr(usage, "server_side_tool_usage_details", None) if not isinstance(details, Mapping): return 0.0 diff --git a/litellm/llms/xai/responses/transformation.py b/litellm/llms/xai/responses/transformation.py index d79e7d4c146..045ea7dcbe9 100644 --- a/litellm/llms/xai/responses/transformation.py +++ b/litellm/llms/xai/responses/transformation.py @@ -1,17 +1,31 @@ -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final + +import httpx import litellm from litellm._logging import verbose_logger from litellm.constants import XAI_API_BASE from litellm.exceptions import AuthenticationError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams +from litellm.types.llms.openai import ( + ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, +) from litellm.types.llms.xai import XAIWebSearchTool, XAIXSearchTool from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as _LiteLLMLoggingObj, + ) + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -250,6 +264,37 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return f"{api_base}/responses" + def transform_response_api_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ResponsesAPIResponse: + """ + Bill what xAI charged instead of repricing the request locally. + + xAI reports the amount on ``usage.cost_in_usd_ticks``; restate it in USD on + ``usage.cost``, which ``ResponseAPILoggingUtils`` already copies onto the chat + Usage that ``llms/xai/cost_calculator.py`` prices, so /v1/responses bills the + reported figure the same way /v1/chat/completions does. When xAI reported nothing + usable, ``cost`` is left alone and the request falls back to token pricing. + """ + response: Final = super().transform_response_api_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + ) + + usage: Final = response.usage + if usage is None: + return response + + reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None)) + if reported_cost is not None: + usage.cost = reported_cost + + return response + def supports_native_websocket(self) -> bool: """XAI does not support native WebSocket for Responses API""" return False diff --git a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py index befd4c5ffbd..905db3d0f2a 100644 --- a/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py +++ b/tests/test_litellm/llms/xai/responses/test_xai_responses_transformation.py @@ -7,12 +7,13 @@ transformations for the Responses API. Source: litellm/llms/xai/responses/transformation.py """ -from unittest.mock import MagicMock - +from unittest.mock import MagicMock, Mock +import httpx import pytest import litellm +from litellm.llms.xai.cost_calculator import cost_per_token from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.llms.openai import ( @@ -400,3 +401,72 @@ class TestXAIResponsesWebSearchBilling: bridged = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(event.response.usage) assert getattr(bridged, "server_side_tool_usage_details") == self._TOOL_DETAILS + + +class TestXAIResponsesReportedCost: + """xAI reports what it charged; the transformation moves it to where litellm bills from. + + ``ResponseAPILoggingUtils`` copies ``usage.cost`` onto the chat Usage that cost + tracking prices, so restating ``cost_in_usd_ticks`` there is what makes /v1/responses + bill the reported figure. At 10^10 ticks to the dollar, 37756000 ticks is $0.0037756. + """ + + @staticmethod + def _transformed_usage(usage: dict) -> ResponseAPIUsage | None: + raw_response = httpx.Response( + status_code=200, + json={ + "id": "resp_xai", + "object": "response", + "created_at": 0, + "model": "grok-4-latest", + "status": "completed", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": usage, + }, + ) + + response = XAIResponsesAPIConfig().transform_response_api_response( + model="grok-4-latest", + raw_response=raw_response, + logging_obj=Mock(), + ) + return response.usage + + def test_reported_cost_reaches_the_cost_calculator(self): + usage = self._transformed_usage( + { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": 37756000, + } + ) + + assert usage.cost == 0.0037756 + + chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) + assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756) + + def test_usage_without_a_reported_cost_is_left_alone(self): + usage = self._transformed_usage( + {"input_tokens": 100, "output_tokens": 200, "total_tokens": 300} + ) + + assert usage.cost is None + + def test_negative_reported_cost_is_not_carried(self): + """A caller who can set api_base must not be able to report negative spend.""" + usage = self._transformed_usage( + { + "input_tokens": 100, + "output_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": -37756000, + } + ) + + assert usage.cost is None diff --git a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py index e5e853ec82f..c67dca11a56 100644 --- a/tests/test_litellm/llms/xai/test_xai_chat_transformation.py +++ b/tests/test_litellm/llms/xai/test_xai_chat_transformation.py @@ -1,9 +1,14 @@ +from unittest.mock import Mock - +import httpx import pytest import litellm -from litellm.llms.xai.chat.transformation import XAIChatConfig +from litellm.llms.xai.chat.transformation import ( + XAIChatCompletionStreamingHandler, + XAIChatConfig, +) +from litellm.llms.xai.cost_calculator import cost_per_token from litellm.types.utils import ( CompletionTokensDetailsWrapper, ModelResponse, @@ -195,3 +200,113 @@ class TestXAIChatWebSearchBilling: ) assert with_search - without_search == pytest.approx(3 * 5.0 / 1000.0) + + +class TestXAIReportedCost: + """xAI reports what it charged; the transformation moves it to where litellm bills from. + + ``cost`` is the field litellm already carries a provider stated cost in, so restating + ``cost_in_usd_ticks`` there is what lets ``llms/xai/cost_calculator.py`` bill the + reported figure. At 10^10 ticks to the dollar, 37756000 ticks is $0.0037756. + """ + + @staticmethod + def _transformed_usage(usage: dict) -> Usage: + raw_response = httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-xai", + "object": "chat.completion", + "created": 0, + "model": "grok-4-latest", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": usage, + }, + ) + + response = XAIChatConfig().transform_response( + model="grok-4-latest", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=Mock(), + request_data={}, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + return response.usage + + def test_reported_cost_reaches_the_cost_calculator(self): + usage = self._transformed_usage( + { + "prompt_tokens": 100, + "completion_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": 37756000, + } + ) + + assert usage.cost == 0.0037756 + assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0037756) + + def test_usage_without_a_reported_cost_is_left_alone(self): + usage = self._transformed_usage( + {"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300} + ) + + assert getattr(usage, "cost", None) is None + + def test_negative_reported_cost_is_not_carried(self): + """A caller who can set api_base must not be able to report negative spend.""" + usage = self._transformed_usage( + { + "prompt_tokens": 100, + "completion_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": -37756000, + } + ) + + assert getattr(usage, "cost", None) is None + + def test_streamed_reported_cost_survives_chunk_aggregation(self): + """Streamed spend only matches if the conversion happens on the chunk. + + Chunk aggregation rebuilds usage from the fields it models plus ``cost``, so a + chunk still carrying only ``cost_in_usd_ticks`` loses the reported amount. + """ + handler = XAIChatCompletionStreamingHandler( + streaming_response=iter([]), sync_stream=True + ) + + parsed = handler.chunk_parser( + { + "id": "chatcmpl-xai", + "object": "chat.completion.chunk", + "created": 0, + "model": "grok-4-latest", + "choices": [], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 200, + "total_tokens": 300, + "cost_in_usd_ticks": 37756000, + }, + } + ) + + assert parsed.usage.cost == 0.0037756 + + assembled = litellm.stream_chunk_builder(chunks=[parsed]) + assert assembled.usage.cost == 0.0037756 + assert cost_per_token(model="grok-4-latest", usage=assembled.usage) == ( + 0.0, + 0.0037756, + ) diff --git a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py index 55e28dff81d..d489ff9fa26 100644 --- a/tests/test_litellm/llms/xai/test_xai_cost_calculator.py +++ b/tests/test_litellm/llms/xai/test_xai_cost_calculator.py @@ -394,6 +394,119 @@ class TestXAICostCalculator: response_object=object(), usage=usage ) + def test_reported_cost_is_preferred_over_token_math(self): + """The amount xAI reported, carried on usage.cost by the transformation, is billed. + + It lands entirely on completion cost because xAI does not split its total by + direction, the same shape the perplexity calculator returns. + """ + usage = Usage( + prompt_tokens=100, + completion_tokens=200, + total_tokens=300, + cost=0.0037756, + ) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost == 0.0 + assert math.isclose(completion_cost, 0.0037756, rel_tol=1e-10) + + def test_reported_cost_suppresses_web_search_surcharge(self): + """The reported total already covers server-side tool calls. + + Without the suppression these 3 searches would be billed a second time on + top of the total xAI already charged. + """ + usage = Usage( + prompt_tokens=100, + completion_tokens=50, + total_tokens=150, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=100, + web_search_requests=3, + ), + cost=0.0037756, + ) + + assert cost_per_web_search_request(usage=usage, model_info={}) == 0.0 + + def test_web_search_surcharge_suppressed_through_the_dispatcher(self): + """The suppression has to hold on the path cost tracking actually uses. + + Legacy behaviour stays intact when xAI reports no cost. + """ + from litellm.llms import get_cost_for_web_search_request + + usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) + + assert get_cost_for_web_search_request("xai", usage, {}) > 0.0 + + reported = Usage( + prompt_tokens=100, completion_tokens=50, total_tokens=150, cost=0.0037756 + ) + setattr(reported, "server_side_tool_usage_details", {"web_search_calls": 3}) + assert get_cost_for_web_search_request("xai", reported, {}) == 0.0 + + def test_no_reported_cost_falls_back_to_token_math(self): + """Absent the provider figure, nothing changes for existing callers.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + + def test_malformed_reported_cost_falls_back_to_token_math(self): + """A junk value must not fail the request, fall back to calculating.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) + setattr(usage, "cost", "not-a-number") + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + + def test_boolean_reported_cost_falls_back_to_token_math(self): + """True is an int in python and would otherwise be billed as $1.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300) + setattr(usage, "cost", True) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + assert completion_cost != 1.0 + + def test_negative_reported_cost_is_rejected(self): + """A negative amount must never reach spend tracking. + + A caller who can set api_base controls the response body, so trusting a + negative figure would let them subtract from their own recorded spend and + slip past a budget. Fall back to token pricing instead, and keep charging + the web search surcharge, since no trustworthy total was reported. + """ + usage = Usage( + prompt_tokens=100, + completion_tokens=200, + total_tokens=300, + cost=-0.0037756, + ) + setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3}) + + prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage) + + assert prompt_cost > 0.0 + assert completion_cost > 0.0 + assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0 + + def test_zero_reported_cost_is_honoured(self): + """A reported zero is a real answer, not a missing value.""" + usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300, cost=0.0) + + assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0) + def test_grok_4_20_beta_reasoning_cost_calculation(self): """Test cost calculation for grok-4.20-beta-0309-reasoning model.""" usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300)