mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge pull request #39441 from BerriAI/litellm_internal_copy_36281
fix(xai): bill from the cost xAI reports instead of recomputing it (internal copy of #36281)
This commit is contained in:
Mateo Wang
GitHub
commit
99da04a1b0
12 changed files with 643 additions and 36 deletions
|
|
@ -4,6 +4,7 @@ import logging
|
|||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
|
||||
from httpx import Response
|
||||
|
|
@ -1164,6 +1165,12 @@ def _store_cost_breakdown_in_logging_obj(
|
|||
# Don't fail the main cost calculation if breakdown storage fails
|
||||
|
||||
|
||||
def _without_provider_stated_cost(usage: Usage | None) -> Usage | None:
|
||||
if usage is None or getattr(usage, "cost", None) is None:
|
||||
return usage
|
||||
return usage.model_copy(update=MappingProxyType({"cost": None}))
|
||||
|
||||
|
||||
def completion_cost(
|
||||
completion_response: object | None = None,
|
||||
model: str | None = None,
|
||||
|
|
@ -1243,7 +1250,10 @@ def completion_cost(
|
|||
cache_creation_input_tokens: int | None = None
|
||||
cache_read_input_tokens: int | None = None
|
||||
audio_transcription_file_duration: float = 0.0
|
||||
cost_per_token_usage_object: Final[Usage | None] = _get_usage_object(completion_response=completion_response)
|
||||
provider_usage_object: Final = _get_usage_object(completion_response=completion_response)
|
||||
cost_per_token_usage_object: Final[Usage | None] = (
|
||||
_without_provider_stated_cost(provider_usage_object) if custom_pricing else provider_usage_object
|
||||
)
|
||||
rerank_billed_units: RerankBilledUnits | None = None
|
||||
|
||||
# Extract service_tier from optional_params if not provided directly
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ FUNCTION_CALL_ATTRIBUTE: Final = "function_call"
|
|||
_SYNC_ITER_EXHAUSTED: Final = object()
|
||||
|
||||
_GCHUNK_FIELDS: Final[frozenset] = frozenset(GChunk.__annotations__)
|
||||
_USAGE_COST_HEADER_PROVIDERS: Final[frozenset[str]] = frozenset({LlmProviders.OPENROUTER.value})
|
||||
|
||||
|
||||
def _next_sync_or_exhausted(it: Any) -> object:
|
||||
|
|
@ -1886,8 +1887,8 @@ class CustomStreamWrapper:
|
|||
@staticmethod
|
||||
def _resolve_provider_reported_cost(usage_cost: object) -> float | None:
|
||||
"""
|
||||
Providers report usage.cost either as a number or, for Perplexity, as a
|
||||
breakdown object whose total lives under ``total_cost``.
|
||||
Providers report usage.cost either as a number or as a breakdown object
|
||||
whose total lives under ``total_cost``.
|
||||
"""
|
||||
if isinstance(usage_cost, bool):
|
||||
return None
|
||||
|
|
@ -1900,12 +1901,10 @@ class CustomStreamWrapper:
|
|||
@staticmethod
|
||||
def _propagate_usage_cost_to_hidden_params(
|
||||
response: "ModelResponse",
|
||||
custom_llm_provider: str | None,
|
||||
) -> None:
|
||||
"""
|
||||
If the assembled response carries a provider-reported cost on
|
||||
usage.cost, copy it into _hidden_params so litellm's cost
|
||||
calculator uses it instead of a token-based estimate.
|
||||
"""
|
||||
if custom_llm_provider not in _USAGE_COST_HEADER_PROVIDERS:
|
||||
return
|
||||
_usage: Final[Usage | None] = getattr(response, "usage", None)
|
||||
_cost: Final = CustomStreamWrapper._resolve_provider_reported_cost(getattr(_usage, "cost", None))
|
||||
if _cost is not None:
|
||||
|
|
@ -2020,7 +2019,7 @@ class CustomStreamWrapper:
|
|||
|
||||
response = self.model_response_creator()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response)
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
||||
setattr(
|
||||
response,
|
||||
|
|
@ -2270,7 +2269,7 @@ class CustomStreamWrapper:
|
|||
|
||||
response: Final = self.model_response_creator()
|
||||
if complete_streaming_response is not None:
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response)
|
||||
self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider)
|
||||
|
||||
setattr(
|
||||
response,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -11,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
filter_value_from_dict,
|
||||
strip_name_from_messages,
|
||||
)
|
||||
from litellm.llms.xai.common_utils import XAIModelInfo
|
||||
from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
|
||||
from litellm.llms.xai.cost_calculator import (
|
||||
apply_server_side_tool_usage_details_to_usage,
|
||||
)
|
||||
|
|
@ -30,6 +31,13 @@ from ...openai.chat.gpt_transformation import (
|
|||
)
|
||||
|
||||
|
||||
def _usage_restated_from_xai_ticks(usage: Usage | None) -> Usage | None:
|
||||
reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None))
|
||||
if usage is None or reported_cost is None:
|
||||
return None
|
||||
return usage.model_copy(update=MappingProxyType({"cost": reported_cost}))
|
||||
|
||||
|
||||
class XAIChatConfig(OpenAIGPTConfig):
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
|
|
@ -283,6 +291,9 @@ class XAIChatConfig(OpenAIGPTConfig):
|
|||
|
||||
self._fold_reasoning_tokens_into_completion(response)
|
||||
self._normalize_openai_compatible_usage_totals(getattr(response, "usage", None))
|
||||
restated_usage: Final = _usage_restated_from_xai_ticks(getattr(response, "usage", None))
|
||||
if restated_usage is not None:
|
||||
response.usage = restated_usage
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -411,4 +422,8 @@ class XAIChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
|
|||
XAIChatConfig._fold_reasoning_tokens_into_completion(chunk["usage"])
|
||||
XAIChatConfig._normalize_openai_compatible_usage_totals(chunk["usage"])
|
||||
|
||||
return super().chunk_parser(chunk)
|
||||
parsed_chunk: Final = super().chunk_parser(chunk)
|
||||
restated_usage: Final = _usage_restated_from_xai_ticks(getattr(parsed_chunk, "usage", None))
|
||||
if restated_usage is not None:
|
||||
parsed_chunk.usage = restated_usage
|
||||
return parsed_chunk
|
||||
|
|
|
|||
|
|
@ -8,6 +8,17 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ProviderSpecificModelInfo
|
||||
|
||||
USD_TICKS_PER_DOLLAR: Final = 10_000_000_000
|
||||
|
||||
|
||||
def xai_reported_cost_in_usd(cost_in_usd_ticks: object) -> float | None:
|
||||
"""xAI bills in ticks of a dollar: https://docs.x.ai/developers/cost-tracking"""
|
||||
if not isinstance(cost_in_usd_ticks, int) or isinstance(cost_in_usd_ticks, bool):
|
||||
return None
|
||||
if cost_in_usd_ticks < 0:
|
||||
return None
|
||||
return cost_in_usd_ticks / USD_TICKS_PER_DOLLAR
|
||||
|
||||
|
||||
class XAIModelInfo(BaseLLMModelInfo):
|
||||
def get_provider_info(
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
"""
|
||||
Helper util for handling XAI-specific cost calculation
|
||||
- Prefers the cost xAI reports on the response over recomputing it locally
|
||||
- Uses the generic cost calculator which already handles tiered pricing correctly
|
||||
- Handles XAI-specific reasoning token billing (billed as part of completion tokens)
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
|
|
@ -36,6 +38,17 @@ def apply_server_side_tool_usage_details_to_usage(usage: Usage, details: Mapping
|
|||
usage.prompt_tokens_details = prompt_tokens_details # rebind-ok: write details onto caller usage
|
||||
|
||||
|
||||
def _cost_reported_by_xai(usage: "Usage") -> float | None:
|
||||
reported_cost: Final[object] = getattr(usage, "cost", None)
|
||||
if not isinstance(reported_cost, (int, float)) or isinstance(reported_cost, bool):
|
||||
return None
|
||||
if not math.isfinite(reported_cost):
|
||||
return None
|
||||
if reported_cost < 0:
|
||||
return None
|
||||
return float(reported_cost)
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given XAI model, prompt tokens, and completion tokens.
|
||||
|
|
@ -48,6 +61,10 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
|
|||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
reported_cost: Final = _cost_reported_by_xai(usage)
|
||||
if reported_cost is not None:
|
||||
return 0.0, reported_cost
|
||||
|
||||
# XAI-specific completion cost: completion is billed as visible + reasoning
|
||||
# tokens. Detect when the transformation layer already folded them so we
|
||||
# don't double-count; fall back to raw xAI shape for callers that bypass
|
||||
|
|
@ -112,6 +129,9 @@ def cost_per_web_search_request(usage: "Usage", model_info: "ModelInfo") -> floa
|
|||
Per-call rate comes from model_info.search_context_cost_per_query when set,
|
||||
otherwise the default xAI tools rate ($5 / 1k calls).
|
||||
"""
|
||||
if _cost_reported_by_xai(usage) is not None:
|
||||
return 0.0
|
||||
|
||||
details: Final = getattr(usage, "server_side_tool_usage_details", None)
|
||||
if not isinstance(details, Mapping):
|
||||
return 0.0
|
||||
|
|
|
|||
|
|
@ -1,17 +1,44 @@
|
|||
from typing import Any, Final
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import XAI_API_BASE
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.llms.xai.common_utils import XAIModelInfo
|
||||
from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
|
||||
from litellm.types.llms.openai import (
|
||||
ResponseAPIUsage,
|
||||
ResponseCompletedEvent,
|
||||
ResponseFailedEvent,
|
||||
ResponseIncompleteEvent,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamingResponse,
|
||||
)
|
||||
from litellm.types.llms.xai import XAIWebSearchTool, XAIXSearchTool
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
Logging as _LiteLLMLoggingObj,
|
||||
)
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
def _usage_restated_from_xai_ticks(usage: ResponseAPIUsage | None) -> ResponseAPIUsage | None:
|
||||
reported_cost: Final = xai_reported_cost_in_usd(getattr(usage, "cost_in_usd_ticks", None))
|
||||
if usage is None or reported_cost is None:
|
||||
return None
|
||||
return usage.model_copy(update=MappingProxyType({"cost": reported_cost}))
|
||||
|
||||
|
||||
class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
||||
"""
|
||||
|
|
@ -250,6 +277,41 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
|
||||
return f"{api_base}/responses"
|
||||
|
||||
def transform_response_api_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ResponsesAPIResponse:
|
||||
response: Final = super().transform_response_api_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
restated_usage: Final = _usage_restated_from_xai_ticks(response.usage)
|
||||
if restated_usage is not None:
|
||||
response.usage = restated_usage
|
||||
return response
|
||||
|
||||
def transform_streaming_response(
|
||||
self,
|
||||
model: str,
|
||||
parsed_chunk: dict, # mutable-ok: overrides the base class signature
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> ResponsesAPIStreamingResponse:
|
||||
event: Final = super().transform_streaming_response(
|
||||
model=model,
|
||||
parsed_chunk=parsed_chunk,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
if not isinstance(event, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent)):
|
||||
return event
|
||||
restated_usage: Final = _usage_restated_from_xai_ticks(event.response.usage)
|
||||
if restated_usage is not None:
|
||||
event.response.usage = restated_usage
|
||||
return event
|
||||
|
||||
def supports_native_websocket(self) -> bool:
|
||||
"""XAI does not support native WebSocket for Responses API"""
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -8595,9 +8595,19 @@ def stream_chunk_builder_text_completion(chunks: list, messages: list | None = N
|
|||
return TextCompletionResponse(**response)
|
||||
|
||||
|
||||
_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value})
|
||||
|
||||
|
||||
def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool:
|
||||
if logging_obj is None:
|
||||
return False
|
||||
provider: Final[object] = logging_obj.model_call_details.get("custom_llm_provider")
|
||||
return provider in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS
|
||||
|
||||
|
||||
def _stream_builder_response_cost(response: ModelResponse, logging_obj: Optional["Logging"]) -> float | None:
|
||||
usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None)
|
||||
if isinstance(usage_cost, (int, float)):
|
||||
if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj):
|
||||
return float(usage_cost)
|
||||
if logging_obj is not None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.litellm_core_utils.streaming_handler import (
|
|||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
Delta,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
PromptTokensDetailsWrapper,
|
||||
StandardLoggingPayload,
|
||||
|
|
@ -1750,7 +1751,7 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params():
|
|||
assert complete_response.usage.cost == 0.00025
|
||||
|
||||
# Use the real propagation method from CustomStreamWrapper
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response)
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openrouter")
|
||||
|
||||
assert "additional_headers" in complete_response._hidden_params
|
||||
assert (
|
||||
|
|
@ -1769,14 +1770,12 @@ def test_openrouter_streaming_cost_propagates_to_hidden_params():
|
|||
assert provider_cost == 0.00025
|
||||
|
||||
|
||||
def test_perplexity_streaming_dict_cost_propagates_to_hidden_params():
|
||||
"""
|
||||
Regression: Perplexity reports usage.cost as a breakdown object, which used to
|
||||
blow up the end of the stream with
|
||||
`float() argument must be a string or a real number, not 'dict'`.
|
||||
"""
|
||||
def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator():
|
||||
import litellm
|
||||
from litellm.cost_calculator import get_response_cost_from_hidden_params
|
||||
from litellm.cost_calculator import (
|
||||
get_response_cost_from_hidden_params,
|
||||
response_cost_calculator,
|
||||
)
|
||||
|
||||
chunks = [
|
||||
ModelResponseStream(
|
||||
|
|
@ -1828,13 +1827,81 @@ def test_perplexity_streaming_dict_cost_propagates_to_hidden_params():
|
|||
|
||||
assert complete_response is not None
|
||||
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response)
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "perplexity")
|
||||
|
||||
assert (
|
||||
get_response_cost_from_hidden_params(complete_response._hidden_params)
|
||||
== 0.00503
|
||||
assert get_response_cost_from_hidden_params(complete_response._hidden_params) is None
|
||||
assert response_cost_calculator(
|
||||
response_object=complete_response,
|
||||
model="perplexity/sonar",
|
||||
custom_llm_provider="perplexity",
|
||||
call_type="completion",
|
||||
optional_params={},
|
||||
) == pytest.approx(0.00503)
|
||||
|
||||
|
||||
def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map():
|
||||
import litellm
|
||||
from litellm.cost_calculator import (
|
||||
get_response_cost_from_hidden_params,
|
||||
response_cost_calculator,
|
||||
)
|
||||
|
||||
model = "openai/streams-cost-in-nanodollars"
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
}
|
||||
}
|
||||
)
|
||||
complete_response = ModelResponse(
|
||||
id="chatcmpl-openai-compatible",
|
||||
model=model,
|
||||
choices=[],
|
||||
usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000),
|
||||
)
|
||||
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai")
|
||||
|
||||
assert get_response_cost_from_hidden_params(complete_response._hidden_params) is None
|
||||
assert response_cost_calculator(
|
||||
response_object=complete_response,
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
call_type="completion",
|
||||
optional_params={},
|
||||
) == pytest.approx(2e-5)
|
||||
|
||||
|
||||
def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch):
|
||||
import litellm
|
||||
from litellm.cost_calculator import (
|
||||
get_response_cost_from_hidden_params,
|
||||
response_cost_calculator,
|
||||
)
|
||||
|
||||
complete_response = ModelResponse(
|
||||
id="chatcmpl-xai",
|
||||
model="grok-4-latest",
|
||||
choices=[],
|
||||
usage=Usage(completion_tokens=353, prompt_tokens=198, total_tokens=551, cost=0.0009956),
|
||||
)
|
||||
|
||||
CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "xai")
|
||||
|
||||
assert get_response_cost_from_hidden_params(complete_response._hidden_params) is None
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5})
|
||||
assert response_cost_calculator(
|
||||
response_object=complete_response,
|
||||
model="xai/grok-4-latest",
|
||||
custom_llm_provider="xai",
|
||||
call_type="completion",
|
||||
optional_params={},
|
||||
) == pytest.approx(0.0009956 * 1.5)
|
||||
|
||||
|
||||
def test_provider_reported_cost_ignores_unusable_shapes():
|
||||
assert CustomStreamWrapper._resolve_provider_reported_cost(None) is None
|
||||
|
|
|
|||
|
|
@ -7,12 +7,13 @@ transformations for the Responses API.
|
|||
Source: litellm/llms/xai/responses/transformation.py
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.xai.cost_calculator import cost_per_token
|
||||
from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -400,3 +401,94 @@ class TestXAIResponsesWebSearchBilling:
|
|||
|
||||
bridged = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(event.response.usage)
|
||||
assert getattr(bridged, "server_side_tool_usage_details") == self._TOOL_DETAILS
|
||||
|
||||
|
||||
class TestXAIResponsesReportedCost:
|
||||
"""xAI reports what it charged; the transformation moves it to where litellm bills from.
|
||||
|
||||
``ResponseAPILoggingUtils`` copies ``usage.cost`` onto the chat Usage that cost
|
||||
tracking prices, so restating ``cost_in_usd_ticks`` there is what makes /v1/responses
|
||||
bill the reported figure. At 10^10 ticks to the dollar, 37756000 ticks is $0.0037756.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _response_body(usage: dict) -> dict:
|
||||
return {
|
||||
"id": "resp_xai",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"model": "grok-4-latest",
|
||||
"status": "completed",
|
||||
"output": [],
|
||||
"parallel_tool_calls": False,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": usage,
|
||||
}
|
||||
|
||||
def _transformed_usage(self, usage: dict) -> ResponseAPIUsage | None:
|
||||
raw_response = httpx.Response(status_code=200, json=self._response_body(usage))
|
||||
|
||||
response = XAIResponsesAPIConfig().transform_response_api_response(
|
||||
model="grok-4-latest",
|
||||
raw_response=raw_response,
|
||||
logging_obj=Mock(),
|
||||
)
|
||||
return response.usage
|
||||
|
||||
def test_reported_cost_reaches_the_cost_calculator(self):
|
||||
usage = self._transformed_usage(
|
||||
{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost_in_usd_ticks": 37756000,
|
||||
}
|
||||
)
|
||||
|
||||
assert usage.cost == 0.0037756
|
||||
|
||||
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
|
||||
assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756)
|
||||
|
||||
def test_streamed_reported_cost_reaches_the_cost_calculator(self):
|
||||
event = XAIResponsesAPIConfig().transform_streaming_response(
|
||||
model="grok-4-latest",
|
||||
parsed_chunk={
|
||||
"type": "response.completed",
|
||||
"sequence_number": 7,
|
||||
"response": self._response_body(
|
||||
{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost_in_usd_ticks": 37756000,
|
||||
}
|
||||
),
|
||||
},
|
||||
logging_obj=Mock(),
|
||||
)
|
||||
|
||||
assert isinstance(event, ResponseCompletedEvent)
|
||||
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(event.response.usage)
|
||||
assert cost_per_token(model="grok-4-latest", usage=chat_usage) == (0.0, 0.0037756)
|
||||
|
||||
def test_usage_without_a_reported_cost_is_left_alone(self):
|
||||
usage = self._transformed_usage(
|
||||
{"input_tokens": 100, "output_tokens": 200, "total_tokens": 300}
|
||||
)
|
||||
|
||||
assert usage.cost is None
|
||||
|
||||
def test_negative_reported_cost_is_not_carried(self):
|
||||
"""A caller who can set api_base must not be able to report negative spend."""
|
||||
usage = self._transformed_usage(
|
||||
{
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost_in_usd_ticks": -37756000,
|
||||
}
|
||||
)
|
||||
|
||||
assert usage.cost is None
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
from unittest.mock import Mock
|
||||
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.xai.chat.transformation import XAIChatConfig
|
||||
from litellm.llms.xai.chat.transformation import (
|
||||
XAIChatCompletionStreamingHandler,
|
||||
XAIChatConfig,
|
||||
)
|
||||
from litellm.llms.xai.cost_calculator import cost_per_token
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
ModelResponse,
|
||||
|
|
@ -195,3 +200,113 @@ class TestXAIChatWebSearchBilling:
|
|||
)
|
||||
|
||||
assert with_search - without_search == pytest.approx(3 * 5.0 / 1000.0)
|
||||
|
||||
|
||||
class TestXAIReportedCost:
|
||||
"""xAI reports what it charged; the transformation moves it to where litellm bills from.
|
||||
|
||||
``cost`` is the field litellm already carries a provider stated cost in, so restating
|
||||
``cost_in_usd_ticks`` there is what lets ``llms/xai/cost_calculator.py`` bill the
|
||||
reported figure. At 10^10 ticks to the dollar, 37756000 ticks is $0.0037756.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _transformed_usage(usage: dict) -> Usage:
|
||||
raw_response = httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-xai",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "grok-4-latest",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "hi"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": usage,
|
||||
},
|
||||
)
|
||||
|
||||
response = XAIChatConfig().transform_response(
|
||||
model="grok-4-latest",
|
||||
raw_response=raw_response,
|
||||
model_response=ModelResponse(),
|
||||
logging_obj=Mock(),
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
return response.usage
|
||||
|
||||
def test_reported_cost_reaches_the_cost_calculator(self):
|
||||
usage = self._transformed_usage(
|
||||
{
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost_in_usd_ticks": 37756000,
|
||||
}
|
||||
)
|
||||
|
||||
assert usage.cost == 0.0037756
|
||||
assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0037756)
|
||||
|
||||
def test_usage_without_a_reported_cost_is_left_alone(self):
|
||||
usage = self._transformed_usage(
|
||||
{"prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300}
|
||||
)
|
||||
|
||||
assert getattr(usage, "cost", None) is None
|
||||
|
||||
def test_negative_reported_cost_is_not_carried(self):
|
||||
"""A caller who can set api_base must not be able to report negative spend."""
|
||||
usage = self._transformed_usage(
|
||||
{
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost_in_usd_ticks": -37756000,
|
||||
}
|
||||
)
|
||||
|
||||
assert getattr(usage, "cost", None) is None
|
||||
|
||||
def test_streamed_reported_cost_survives_chunk_aggregation(self):
|
||||
"""Streamed spend only matches if the conversion happens on the chunk.
|
||||
|
||||
Chunk aggregation rebuilds usage from the fields it models plus ``cost``, so a
|
||||
chunk still carrying only ``cost_in_usd_ticks`` loses the reported amount.
|
||||
"""
|
||||
handler = XAIChatCompletionStreamingHandler(
|
||||
streaming_response=iter([]), sync_stream=True
|
||||
)
|
||||
|
||||
parsed = handler.chunk_parser(
|
||||
{
|
||||
"id": "chatcmpl-xai",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "grok-4-latest",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 200,
|
||||
"total_tokens": 300,
|
||||
"cost_in_usd_ticks": 37756000,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert parsed.usage.cost == 0.0037756
|
||||
|
||||
assembled = litellm.stream_chunk_builder(chunks=[parsed])
|
||||
assert assembled.usage.cost == 0.0037756
|
||||
assert cost_per_token(model="grok-4-latest", usage=assembled.usage) == (
|
||||
0.0,
|
||||
0.0037756,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@ import os
|
|||
|
||||
import litellm
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
CompletionTokensDetailsWrapper,
|
||||
Message,
|
||||
ModelResponse,
|
||||
PromptTokensDetailsWrapper,
|
||||
Usage,
|
||||
)
|
||||
|
|
@ -361,6 +364,145 @@ class TestXAICostCalculator:
|
|||
response_object=object(), usage=usage
|
||||
)
|
||||
|
||||
def test_reported_cost_is_preferred_over_token_math(self):
|
||||
"""The amount xAI reported, carried on usage.cost by the transformation, is billed.
|
||||
|
||||
It lands entirely on completion cost because xAI does not split its total by
|
||||
direction, the same shape the perplexity calculator returns.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=200,
|
||||
total_tokens=300,
|
||||
cost=0.0037756,
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage)
|
||||
|
||||
assert prompt_cost == 0.0
|
||||
assert math.isclose(completion_cost, 0.0037756, rel_tol=1e-10)
|
||||
|
||||
def test_reported_cost_suppresses_web_search_surcharge(self):
|
||||
"""The reported total already covers server-side tool calls.
|
||||
|
||||
Without the suppression these 3 searches would be billed a second time on
|
||||
top of the total xAI already charged.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=50,
|
||||
total_tokens=150,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=100,
|
||||
web_search_requests=3,
|
||||
),
|
||||
cost=0.0037756,
|
||||
)
|
||||
|
||||
assert cost_per_web_search_request(usage=usage, model_info={}) == 0.0
|
||||
|
||||
def test_web_search_surcharge_suppressed_through_the_dispatcher(self):
|
||||
"""The suppression has to hold on the path cost tracking actually uses.
|
||||
|
||||
Legacy behaviour stays intact when xAI reports no cost.
|
||||
"""
|
||||
from litellm.llms import get_cost_for_web_search_request
|
||||
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3})
|
||||
|
||||
assert get_cost_for_web_search_request("xai", usage, {}) > 0.0
|
||||
|
||||
reported = Usage(
|
||||
prompt_tokens=100, completion_tokens=50, total_tokens=150, cost=0.0037756
|
||||
)
|
||||
setattr(reported, "server_side_tool_usage_details", {"web_search_calls": 3})
|
||||
assert get_cost_for_web_search_request("xai", reported, {}) == 0.0
|
||||
|
||||
def test_no_reported_cost_falls_back_to_token_math(self):
|
||||
"""Absent the provider figure, nothing changes for existing callers."""
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage)
|
||||
|
||||
assert prompt_cost > 0.0
|
||||
assert completion_cost > 0.0
|
||||
|
||||
def test_malformed_reported_cost_falls_back_to_token_math(self):
|
||||
"""A junk value must not fail the request, fall back to calculating."""
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300)
|
||||
setattr(usage, "cost", "not-a-number")
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage)
|
||||
|
||||
assert prompt_cost > 0.0
|
||||
assert completion_cost > 0.0
|
||||
|
||||
def test_boolean_reported_cost_falls_back_to_token_math(self):
|
||||
"""True is an int in python and would otherwise be billed as $1."""
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300)
|
||||
setattr(usage, "cost", True)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage)
|
||||
|
||||
assert prompt_cost > 0.0
|
||||
assert completion_cost > 0.0
|
||||
assert completion_cost != 1.0
|
||||
|
||||
def test_negative_reported_cost_is_rejected(self):
|
||||
"""A negative amount must never reach spend tracking.
|
||||
|
||||
A caller who can set api_base controls the response body, so trusting a
|
||||
negative figure would let them subtract from their own recorded spend and
|
||||
slip past a budget. Fall back to token pricing instead, and keep charging
|
||||
the web search surcharge, since no trustworthy total was reported.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=200,
|
||||
total_tokens=300,
|
||||
cost=-0.0037756,
|
||||
)
|
||||
setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3})
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage)
|
||||
|
||||
assert prompt_cost > 0.0
|
||||
assert completion_cost > 0.0
|
||||
assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0
|
||||
|
||||
def test_non_finite_reported_cost_is_rejected(self):
|
||||
"""NaN compares false against every budget threshold.
|
||||
|
||||
Usage stores a provider supplied cost without validating it, so a caller who
|
||||
controls the response body could report NaN and leave spend >= max_budget
|
||||
false for the life of the key rather than mispricing one request. The
|
||||
infinities are refused alongside it. Fall back to token pricing and keep
|
||||
charging the web search surcharge, since no trustworthy total was reported.
|
||||
"""
|
||||
for reported_cost in (float("nan"), float("inf"), float("-inf")):
|
||||
usage = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=200,
|
||||
total_tokens=300,
|
||||
cost=reported_cost,
|
||||
)
|
||||
setattr(usage, "server_side_tool_usage_details", {"web_search_calls": 3})
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="grok-4-latest", usage=usage)
|
||||
|
||||
assert math.isfinite(prompt_cost), reported_cost
|
||||
assert math.isfinite(completion_cost), reported_cost
|
||||
assert prompt_cost > 0.0, reported_cost
|
||||
assert completion_cost > 0.0, reported_cost
|
||||
assert cost_per_web_search_request(usage=usage, model_info={}) > 0.0, reported_cost
|
||||
|
||||
def test_zero_reported_cost_is_honoured(self):
|
||||
"""A reported zero is a real answer, not a missing value."""
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300, cost=0.0)
|
||||
|
||||
assert cost_per_token(model="grok-4-latest", usage=usage) == (0.0, 0.0)
|
||||
|
||||
def test_grok_4_20_beta_reasoning_cost_calculation(self):
|
||||
"""Test cost calculation for grok-4.20-beta-0309-reasoning model."""
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=200, total_tokens=300)
|
||||
|
|
@ -437,6 +579,48 @@ class TestXAICostCalculator:
|
|||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
def test_custom_pricing_beats_the_reported_cost(self):
|
||||
response = ModelResponse(
|
||||
id="chatcmpl-xai",
|
||||
model="grok-4-latest",
|
||||
choices=[Choices(index=0, message=Message(role="assistant", content="x"), finish_reason="stop")],
|
||||
usage=Usage(prompt_tokens=198, completion_tokens=353, total_tokens=551, cost=0.0009956),
|
||||
)
|
||||
|
||||
billed = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="xai/grok-4-latest",
|
||||
custom_llm_provider="xai",
|
||||
custom_cost_per_token={"input_cost_per_token": 0.001, "output_cost_per_token": 0.001},
|
||||
custom_pricing=True,
|
||||
)
|
||||
|
||||
assert math.isclose(billed, 0.551, rel_tol=1e-10)
|
||||
|
||||
def test_deployment_custom_pricing_beats_the_reported_cost(self, monkeypatch):
|
||||
deployment_id = "xai-deployment-priced-by-the-operator"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
deployment_id,
|
||||
{"input_cost_per_token": 0.001, "output_cost_per_token": 0.001, "litellm_provider": "xai", "mode": "chat"},
|
||||
)
|
||||
response = ModelResponse(
|
||||
id="chatcmpl-xai",
|
||||
model="grok-4-latest",
|
||||
choices=[Choices(index=0, message=Message(role="assistant", content="x"), finish_reason="stop")],
|
||||
usage=Usage(prompt_tokens=198, completion_tokens=353, total_tokens=551, cost=0.0009956),
|
||||
)
|
||||
|
||||
billed = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="xai/grok-4-latest",
|
||||
custom_llm_provider="xai",
|
||||
custom_pricing=True,
|
||||
router_model_id=deployment_id,
|
||||
)
|
||||
|
||||
assert math.isclose(billed, 0.551, rel_tol=1e-10)
|
||||
|
||||
|
||||
class TestXAIWebSearchCostHelpers:
|
||||
"""Focused coverage for web_search / tool-usage helpers in cost_calculator.py."""
|
||||
|
|
|
|||
|
|
@ -3131,9 +3131,9 @@ def test_stream_chunk_builder_prices_proxy_alias_via_model_map():
|
|||
assert response._hidden_params["response_cost"] == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def _stream_builder_logging_obj() -> LiteLLMLogging:
|
||||
def _stream_builder_logging_obj(model: str = "gpt-4o", custom_llm_provider: str = "openai") -> LiteLLMLogging:
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="gpt-4o",
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
|
|
@ -3142,10 +3142,11 @@ def _stream_builder_logging_obj() -> LiteLLMLogging:
|
|||
function_id="test-function-id",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="gpt-4o",
|
||||
model=model,
|
||||
user=None,
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "openai"},
|
||||
litellm_params={"custom_llm_provider": custom_llm_provider},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
|
@ -3237,3 +3238,24 @@ def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk():
|
|||
assert response.usage.completion_tokens == 60
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.000704)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.000704)
|
||||
|
||||
|
||||
def test_stream_chunk_builder_leaves_xai_reported_cost_to_the_calculator(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "cost_margin_config", {"xai": 0.5})
|
||||
usage_chunk: Final = _stream_builder_text_chunk("grok-4", "")
|
||||
usage_chunk.usage = Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7, cost=0.42)
|
||||
chunks: Final = [
|
||||
_stream_builder_text_chunk("grok-4", "Hello "),
|
||||
_stream_builder_text_chunk("grok-4", "world.", finish_reason="stop"),
|
||||
usage_chunk,
|
||||
]
|
||||
logging_obj: Final = _stream_builder_logging_obj(model="grok-4", custom_llm_provider="xai")
|
||||
|
||||
response: Final = litellm.stream_chunk_builder(
|
||||
chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj
|
||||
)
|
||||
|
||||
assert response is not None
|
||||
assert getattr(response.usage, "cost", None) == pytest.approx(0.42)
|
||||
assert response._hidden_params.get("response_cost") is None
|
||||
assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.63)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue