From efd827431518f2474e6288d43bcca4d4d4a4e3fb Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Wed, 14 Oct 2026 22:27:02 -0700 Subject: [PATCH] feat(xai): bill grok voice realtime sessions (#46600) * feat(xai): bill grok voice realtime sessions * chore: revert unrelated formatting churn in test_cost_calculator.py * refactor(xai): tidy realtime usage meter and tests * fix(xai): read push-to-talk mode from client session.update only * fix(xai): clamp decoded audio bytes and lock push-to-talk before content --------- Co-authored-by: kerry --- litellm/constants.py | 1 + litellm/cost_calculator.py | 63 ++++- .../litellm_core_utils/realtime_streaming.py | 23 +- litellm/llms/openai/realtime/handler.py | 6 + litellm/llms/xai/realtime/handler.py | 6 + litellm/llms/xai/realtime/usage_meter.py | 163 +++++++++++++ litellm/types/llms/xai.py | 11 + litellm/types/router.py | 1 + litellm/types/utils.py | 3 + litellm/utils.py | 1 + .../test_realtime_streaming.py | 63 ++++- .../llms/xai/test_xai_realtime_usage_meter.py | 216 ++++++++++++++++++ tests/unit/test_cost_calculator.py | 174 +++++++++++++- tests/unit/test_utils.py | 2 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 15 files changed, 723 insertions(+), 14 deletions(-) create mode 100644 litellm/llms/xai/realtime/usage_meter.py create mode 100644 tests/unit/llms/xai/test_xai_realtime_usage_meter.py diff --git a/litellm/constants.py b/litellm/constants.py index c72e7ab777b..3e855d0bfbf 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -397,6 +397,7 @@ MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolve MCP_PEEKED_BODY_SCOPE_KEY: Final = "litellm_mcp_peeked_body" MCP_ADMISSION_BODY_PEEK_TIMEOUT_SECONDS: Final = 5.0 REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged" +REALTIME_METERED_USAGE_KEY: Final = "realtime_metered_usage" OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS: Final = 30 OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS: Final = 5 OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset( diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 918b95280be..245f65eb69c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -20,6 +20,7 @@ from litellm.constants import ( DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, + REALTIME_METERED_USAGE_KEY, ) from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.cost_discount import resolve_cost_discount @@ -117,6 +118,7 @@ from litellm.types.llms.openai import ( ResponseAPIUsage, ResponsesAPIResponse, ) +from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankBilledUnits, RerankResponse from litellm.types.utils import ( CallTypesLiteral, @@ -828,7 +830,14 @@ def _get_hidden_str_for_cost_calc(hidden_params: object, key: str) -> str | None _NON_TOKEN_RATE_FIELDS: Final = frozenset( - {"cost_per_second", "input_cost_per_second", "output_cost_per_second", "input_cost_per_query", "tiered_pricing"} + { + "cost_per_second", + "input_cost_per_second", + "output_cost_per_second", + "input_cost_per_query", + "input_cost_per_text_message", + "tiered_pricing", + } ) @@ -3119,31 +3128,43 @@ def _live_session_wall_clock_seconds(litellm_logging_obj: LitellmLoggingObject | return 0.0 -def _live_input_cost_per_second(model_name: str, custom_llm_provider: str) -> float | None: +def _live_model_rate( + model_name: str, + custom_llm_provider: str, + rate_field: Literal["input_cost_per_second", "input_cost_per_text_message"], +) -> float | None: model_info: Final = _lookup_model_info_or_none(model_name, custom_llm_provider) if model_info is None: return None - rate: Final = model_info.get("input_cost_per_second") + rate: Final = model_info.get(rate_field) if isinstance(rate, (int, float)) and not isinstance(rate, bool) and math.isfinite(rate): return float(rate) return None -def _first_live_input_cost_per_second( +def _first_live_model_rate( potential_model_names: Sequence[str | None], custom_llm_provider: str, + rate_field: Literal["input_cost_per_second", "input_cost_per_text_message"], ) -> float: return next( ( rate for model_name in potential_model_names if model_name is not None - and (rate := _live_input_cost_per_second(model_name, custom_llm_provider)) is not None + and (rate := _live_model_rate(model_name, custom_llm_provider, rate_field)) is not None ), 0.0, ) +def _xai_metered_usage(litellm_logging_obj: LitellmLoggingObject | None) -> RealtimeMeteredUsage | None: + if litellm_logging_obj is None: + return None + usage: Final = litellm_logging_obj.model_call_details.get(REALTIME_METERED_USAGE_KEY) + return usage if isinstance(usage, RealtimeMeteredUsage) else None + + def _live_response_token_costs( response: _LiveResponsePayload, custom_llm_provider: str, @@ -3303,9 +3324,28 @@ def handle_realtime_stream_cost_calculation( live_model_names: Final = ( (custom_pricing_model, base_pricing_model, received_model, litellm_model_name) if live_event_seen else () ) - live_voice_cost: Final = voice_seconds * _first_live_input_cost_per_second( + live_voice_cost: Final = voice_seconds * _first_live_model_rate( live_model_names, custom_llm_provider, + "input_cost_per_second", + ) + xai_metered: Final = ( + _xai_metered_usage(litellm_logging_obj) if custom_llm_provider == LlmProviders.XAI.value else None + ) + xai_audio_seconds: Final = ( + xai_metered.audio_seconds_sent + xai_metered.audio_seconds_received + if xai_metered is not None and xai_metered.push_to_talk + else _live_session_wall_clock_seconds(litellm_logging_obj) + if custom_llm_provider == LlmProviders.XAI.value + else 0.0 + ) + audio_duration_cost: Final = xai_audio_seconds * _first_live_model_rate( + potential_model_names, + custom_llm_provider, + "input_cost_per_second", + ) + text_message_cost: Final = (xai_metered.text_messages if xai_metered is not None else 0) * _first_live_model_rate( + potential_model_names, custom_llm_provider, "input_cost_per_text_message" ) live_response_costs: Final = tuple( _live_response_token_costs( @@ -3319,10 +3359,17 @@ def handle_realtime_stream_cost_calculation( live_response_output_cost: Final = sum(costs[1] for costs in live_response_costs) input_cost: Final = input_cost_per_token + live_response_input_cost output_cost: Final = output_cost_per_token + live_response_output_cost - total_cost: Final = input_cost + output_cost + transcription_cost + live_voice_cost + total_cost: Final = ( + input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost + ) additional_costs: Final = tuple( (cost_name, cost) - for cost_name, cost in (("transcription_cost", transcription_cost), ("live_voice_cost", live_voice_cost)) + for cost_name, cost in ( + ("transcription_cost", transcription_cost), + ("live_voice_cost", live_voice_cost), + ("audio_duration_cost", audio_duration_cost), + ("text_message_cost", text_message_cost), + ) if cost > 0 ) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index c581db6370b..0b659b31d00 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -10,7 +10,11 @@ from typing_extensions import ReadOnly import litellm from litellm._logging import redact_internal_details_from_client_message, verbose_logger -from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY +from litellm.constants import ( + REALTIME_METERED_USAGE_KEY, + REALTIME_SESSION_FAILURE_LOGGED_KEY, + REALTIME_SESSION_SUCCESS_LOGGED_KEY, +) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( @@ -114,6 +118,12 @@ class RealtimeEventNormalizer(Protocol): def patch_outgoing_session(self, session: dict) -> dict: ... +class RealtimeUsageMeter(Protocol): + def observe_client_event(self, event: Mapping[str, object]) -> None: ... + def observe_backend_event(self, event: Mapping[str, object]) -> None: ... + def snapshot(self) -> object: ... + + DefaultLoggedRealTimeEventTypes: Final = [ "session.created", "response.create", @@ -136,6 +146,7 @@ class RealTimeStreaming: backend_uses_beta_protocol: bool | None = None, force_transcription_model: str | None = None, event_normalizer: RealtimeEventNormalizer | None = None, + usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, ): self.websocket: _ClientWebSocket = websocket @@ -198,6 +209,7 @@ class RealTimeStreaming: self._is_transcription_session: bool = force_transcription_model is not None # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer). self._event_normalizer = event_normalizer + self._usage_meter = usage_meter @property def _session_created_sent_to_client(self) -> bool: @@ -416,6 +428,8 @@ class RealTimeStreaming: async def log_messages(self): """Log messages in list""" if self.logging_obj: + if self._usage_meter is not None: + self.logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] = self._usage_meter.snapshot() if self.input_messages: self.logging_obj.model_call_details["messages"] = self.input_messages if self.session_tools or self.tool_calls: @@ -481,6 +495,10 @@ class RealTimeStreaming: sent = True return sent await self.backend_ws.send(message) + if self._usage_meter is not None: + sent_event: Final = self._parse_backend_event(message) + if sent_event is not None: + self._usage_meter.observe_client_event(sent_event) return True def _enforce_transcription_session_model(self, message: str) -> str: @@ -1113,6 +1131,9 @@ class RealTimeStreaming: if self._should_drop_event_from_client(event): continue + if self._usage_meter is not None: + self._usage_meter.observe_backend_event(event) + if await self._handle_raw_backend_message(event, raw_response): continue diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 853f707bc91..080fc3d24c9 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -16,6 +16,7 @@ from ....litellm_core_utils.realtime_errors import close_after_upstream_handshak from ....litellm_core_utils.realtime_streaming import ( RealtimeEventNormalizer, RealTimeStreaming, + RealtimeUsageMeter, client_sent_openai_beta_realtime_header, ) from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context @@ -123,6 +124,10 @@ class OpenAIRealtime(OpenAIChatCompletion): """ return None + def _make_usage_meter(self) -> RealtimeUsageMeter | None: + """Return a per-session usage meter, or None when billing needs no metering.""" + return None + async def async_realtime( self, model: str, @@ -190,6 +195,7 @@ class OpenAIRealtime(OpenAIChatCompletion): model if (query_params or {}).get("intent") == "transcription" else None ), event_normalizer=self._make_event_normalizer(), + usage_meter=self._make_usage_meter(), ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/xai/realtime/handler.py b/litellm/llms/xai/realtime/handler.py index 9dac5d945fd..5ff487bea72 100644 --- a/litellm/llms/xai/realtime/handler.py +++ b/litellm/llms/xai/realtime/handler.py @@ -8,9 +8,11 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ from litellm.constants import XAI_API_BASE +from litellm.litellm_core_utils.realtime_streaming import RealtimeUsageMeter from ...openai.realtime.handler import OpenAIRealtime from .transformation import XAIRealtimeNormalizer +from .usage_meter import XAIRealtimeUsageMeter class XAIRealtime(OpenAIRealtime): @@ -33,6 +35,10 @@ class XAIRealtime(OpenAIRealtime): """Return a fresh per-session XAI normalizer instance.""" return XAIRealtimeNormalizer() + def _make_usage_meter(self) -> RealtimeUsageMeter: + """xAI bills realtime audio per second and text per client message.""" + return XAIRealtimeUsageMeter() + def _get_additional_headers( self, api_key: str, diff --git a/litellm/llms/xai/realtime/usage_meter.py b/litellm/llms/xai/realtime/usage_meter.py new file mode 100644 index 00000000000..00e8a402544 --- /dev/null +++ b/litellm/llms/xai/realtime/usage_meter.py @@ -0,0 +1,163 @@ +"""Counts billable xAI Grok Voice realtime usage from raw wire frames. + +Audio format reference: https://docs.x.ai/voice-realtime.ws.json +""" + +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.types.llms.xai import RealtimeMeteredUsage + +_PCM_DEFAULT_RATE: Final = 24000 +_G711_BYTES_PER_SECOND: Final = 8000 +_AUDIO_CONTENT_PART_TYPES: Final = frozenset({"input_audio", "audio"}) +_CONTENT_PART_LIST: Final = TypeAdapter(list[object]) + + +def _decoded_base64_length(value: object) -> int: + """Decoded byte length of a base64 string without decoding it.""" + if not isinstance(value, str): + return 0 + padding: Final = min(len(value) - len(value.rstrip("=")), 2) + return max(len(value) * 3 // 4 - padding, 0) + + +def _audio_format_bytes_per_second(fmt: Mapping[str, object]) -> float: + fmt_type: Final = fmt.get("type") + if fmt_type is None or fmt_type == "audio/pcm": + rate: Final = fmt.get("rate") + rate_hz: Final = ( + int(rate) + if isinstance(rate, (int, float)) and not isinstance(rate, bool) and rate > 0 + else _PCM_DEFAULT_RATE + ) + return float(2 * rate_hz) + if fmt_type in ("audio/pcmu", "audio/pcma"): + return float(_G711_BYTES_PER_SECOND) + return 0.0 + + +def _beta_audio_format_bytes_per_second(name: object) -> float | None: + if name == "pcm16": + return float(2 * _PCM_DEFAULT_RATE) + if name in ("g711_ulaw", "g711_alaw"): + return float(_G711_BYTES_PER_SECOND) + return None + + +def _content_part_mappings(item: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + content: Final = item.get("content") + raw_parts: Final = _CONTENT_PART_LIST.validate_python(content) if isinstance(content, list) else () + return tuple(part_map for part in raw_parts if (part_map := as_str_mapping(part)) is not None) + + +def _is_billable_text_item(item: Mapping[str, object], parts: tuple[Mapping[str, object], ...]) -> bool: + if item.get("type") == "function_call_output": + return False + return not any(part.get("type") in _AUDIO_CONTENT_PART_TYPES for part in parts) + + +class XAIRealtimeUsageMeter: + """Counts billable xAI realtime usage from raw client and backend frames.""" + + def __init__(self) -> None: + self._push_to_talk: bool = False + self._audio_seconds_sent: float = 0.0 + self._audio_seconds_received: float = 0.0 + self._text_messages: int = 0 + self._content_sent: bool = False + self._input_bytes_per_second: float = float(2 * _PCM_DEFAULT_RATE) + self._output_bytes_per_second: float = float(2 * _PCM_DEFAULT_RATE) + + def observe_client_event(self, event: Mapping[str, object]) -> None: + event_type: Final = event.get("type") + if event_type == "session.update": + session: Final = as_str_mapping(event.get("session")) + if session is not None: + self._apply_client_turn_detection(session) + self._apply_session_config(session) + elif event_type == "input_audio_buffer.append": + self._content_sent = True + if self._input_bytes_per_second > 0: + self._audio_seconds_sent += _decoded_base64_length(event.get("audio")) / self._input_bytes_per_second + elif event_type == "conversation.item.create": + self._content_sent = True + item: Final = as_str_mapping(event.get("item")) + if item is not None: + parts: Final = _content_part_mappings(item) + if _is_billable_text_item(item, parts): + self._text_messages += 1 + if self._input_bytes_per_second > 0: + self._audio_seconds_sent += ( + sum( + _decoded_base64_length(part.get("audio")) + for part in parts + if part.get("type") in _AUDIO_CONTENT_PART_TYPES + ) + / self._input_bytes_per_second + ) + + def observe_backend_event(self, event: Mapping[str, object]) -> None: + event_type: Final = event.get("type") + if event_type in ("session.created", "session.updated"): + session: Final = as_str_mapping(event.get("session")) + if session is not None: + self._apply_session_config(session) + elif event_type in ("response.output_audio.delta", "response.audio.delta"): + if self._output_bytes_per_second > 0: + self._audio_seconds_received += ( + _decoded_base64_length(event.get("delta")) / self._output_bytes_per_second + ) + + def snapshot(self) -> RealtimeMeteredUsage: + return RealtimeMeteredUsage( + push_to_talk=self._push_to_talk, + audio_seconds_sent=self._audio_seconds_sent, + audio_seconds_received=self._audio_seconds_received, + text_messages=self._text_messages, + ) + + def _apply_session_config(self, session: Mapping[str, object]) -> None: + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + + input_bps: Final = _beta_audio_format_bytes_per_second(session.get("input_audio_format")) + if input_bps is not None: + self._input_bytes_per_second = input_bps + output_bps: Final = _beta_audio_format_bytes_per_second(session.get("output_audio_format")) + if output_bps is not None: + self._output_bytes_per_second = output_bps + + if audio is None: + return + if audio_input is not None: + input_format: Final = as_str_mapping(audio_input.get("format")) + if input_format is not None: + self._input_bytes_per_second = _audio_format_bytes_per_second(input_format) + output_config: Final = as_str_mapping(audio.get("output")) + if output_config is not None: + output_format: Final = as_str_mapping(output_config.get("format")) + if output_format is not None: + self._output_bytes_per_second = _audio_format_bytes_per_second(output_format) + + def _apply_client_turn_detection(self, session: Mapping[str, object]) -> None: + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + if "turn_detection" in session: + self._apply_turn_detection(session["turn_detection"]) + elif audio_input is not None and "turn_detection" in audio_input: + self._apply_turn_detection(audio_input["turn_detection"]) + + def _apply_turn_detection(self, value: object) -> None: + config: Final = as_str_mapping(value) + if value is not None and (config is None or "type" not in config): + return + push_to_talk: Final = value is None or (config is not None and config["type"] is None) + # Switching to push-to-talk after content was sent keeps wall-clock + # billing, so a late session.update cannot discount an elapsed VAD session + if push_to_talk and self._content_sent: + return + self._push_to_talk = push_to_talk diff --git a/litellm/types/llms/xai.py b/litellm/types/llms/xai.py index ec711f6d042..000b0cf5a0b 100644 --- a/litellm/types/llms/xai.py +++ b/litellm/types/llms/xai.py @@ -1,6 +1,17 @@ +from dataclasses import dataclass from typing import Literal, TypedDict +@dataclass(frozen=True, slots=True) +class RealtimeMeteredUsage: + """Per-session xAI realtime usage counted from raw wire frames.""" + + push_to_talk: bool + audio_seconds_sent: float + audio_seconds_received: float + text_messages: int + + class XAIWebSearchFilters(TypedDict, total=False): """Filters for XAI web search tool""" diff --git a/litellm/types/router.py b/litellm/types/router.py index 285a97c3221..688ed29b265 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -734,6 +734,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): output_cost_per_token: float | None cost_per_second: ReadOnly[float | None] input_cost_per_second: float | None + input_cost_per_text_message: ReadOnly[float | None] output_cost_per_second: float | None output_cost_per_second_480p: ReadOnly[float | None] output_cost_per_second_720p: ReadOnly[float | None] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 67f4c653b24..d5b8a7c7db9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -355,6 +355,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_image_token_batches: ReadOnly[float | None] cost_per_second: ReadOnly[float | None] input_cost_per_second: float | None # for OpenAI Speech models + input_cost_per_text_message: ReadOnly[float | None] input_cost_per_token_batches: float | None input_cost_per_video_token_batches: ReadOnly[float | None] input_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None] @@ -2885,6 +2886,7 @@ class LoggedLiteLLMParams(TypedDict, total=False): no_log: bool | None cost_per_second: ReadOnly[float | None] input_cost_per_second: float | None + input_cost_per_text_message: ReadOnly[float | None] input_cost_per_token: float | None output_cost_per_token: float | None output_cost_per_second: float | None @@ -3857,6 +3859,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): ## CUSTOM PRICING ## cost_per_second: float | None = None input_cost_per_second: float | None = None + input_cost_per_text_message: float | None = None output_cost_per_second: float | None = None output_cost_per_second_1080p: float | None = None output_cost_per_second_480p: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 5d1372570a9..596fd96b021 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6344,6 +6344,7 @@ def get_model_info_helper( input_cost_per_query=_model_info.get("input_cost_per_query", None), cost_per_second=_model_info.get("cost_per_second", None), input_cost_per_second=_model_info.get("input_cost_per_second", None), + input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None), input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None), diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index fd72af47e09..8de5e27d99c 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -1,4 +1,5 @@ import asyncio +import base64 import json from collections.abc import Coroutine, Mapping, Sequence from dataclasses import dataclass @@ -11,7 +12,11 @@ from websockets.exceptions import ConnectionClosed from websockets.frames import Close import litellm -from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY +from litellm.constants import ( + REALTIME_METERED_USAGE_KEY, + REALTIME_SESSION_FAILURE_LOGGED_KEY, + REALTIME_SESSION_SUCCESS_LOGGED_KEY, +) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.realtime_streaming import ( RealTimeStreaming, @@ -20,7 +25,9 @@ from litellm.litellm_core_utils.realtime_streaming import ( from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer +from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeResponseTypedDict from litellm.types.utils import GenericGuardrailAPIInputs @@ -784,6 +791,60 @@ def test_collect_user_input_empty_transcript_not_collected(): assert len(streaming.input_messages) == 0 +@pytest.mark.asyncio +async def test_usage_meter_snapshot_stored_in_model_call_details(): + """xAI metered usage from raw client/backend frames reaches the spend log.""" + client_ws = MagicMock() + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock( + side_effect=[ + json.dumps( + {"type": "session.created", "session": {"turn_detection": {"type": None}}} + ).encode(), + json.dumps( + {"type": "response.output_audio.delta", "delta": base64.b64encode(b"x" * 24000).decode()} + ).encode(), + ConnectionClosed(None, None), + ] + ) + logging_obj: Final = _mock_logging() + logging_obj.model_call_details = {} + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + logging_obj.dispatch_success_handlers = AsyncMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + event_normalizer=XAIRealtimeNormalizer(), + usage_meter=XAIRealtimeUsageMeter(), + ) + + await streaming._send_to_backend(json.dumps({"type": "session.update", "session": {"turn_detection": None}})) + await streaming._send_to_backend( + json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"y" * 48000).decode()}) + ) + await streaming._send_to_backend( + json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + } + ) + ) + await streaming.backend_to_client_send_messages() + await streaming.log_messages() + + metered = logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] + assert isinstance(metered, RealtimeMeteredUsage) + assert metered.push_to_talk is True + assert metered.audio_seconds_sent == 1.0 + assert metered.audio_seconds_received == 0.5 + assert metered.text_messages == 1 + + @pytest.mark.asyncio async def test_log_messages_sets_input_messages_on_logging_obj(): """ diff --git a/tests/unit/llms/xai/test_xai_realtime_usage_meter.py b/tests/unit/llms/xai/test_xai_realtime_usage_meter.py new file mode 100644 index 00000000000..907d23b659c --- /dev/null +++ b/tests/unit/llms/xai/test_xai_realtime_usage_meter.py @@ -0,0 +1,216 @@ +import base64 + +from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter + + +def _b64(num_bytes: int) -> str: + return base64.b64encode(b"x" * num_bytes).decode() + + +def test_text_messages_counts_only_billable_items(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + } + ) + meter.observe_client_event( + {"type": "conversation.item.create", "item": {"type": "function_call_output", "output": "{}"}} + ) + meter.observe_client_event( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_audio", "audio": _b64(10)}]}, + } + ) + meter.observe_client_event({"type": "response.create"}) + + assert meter.snapshot().text_messages == 1 + + +def test_input_audio_item_part_counts_as_sent_audio_not_text(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_audio", "audio": _b64(48000)}], + }, + } + ) + + snapshot = meter.snapshot() + assert snapshot.audio_seconds_sent == 1.0 + assert snapshot.text_messages == 0 + + +def test_audio_seconds_default_pcm24(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)}) + meter.observe_backend_event({"type": "response.output_audio.delta", "delta": _b64(24000)}) + meter.observe_backend_event({"type": "response.audio.delta", "delta": _b64(24000)}) + + snapshot = meter.snapshot() + assert snapshot.audio_seconds_sent == 1.0 + assert snapshot.audio_seconds_received == 1.0 + + +def test_session_update_nested_pcm_rate_changes_seconds(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event( + { + "type": "session.update", + "session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}}, + } + ) + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(32000)}) + + assert meter.snapshot().audio_seconds_sent == 1.0 + + +def test_g711_format_bills_at_8000_bytes_per_second(): + meter = XAIRealtimeUsageMeter() + + meter.observe_backend_event( + { + "type": "session.created", + "session": {"audio": {"output": {"format": {"type": "audio/pcmu"}}}}, + } + ) + meter.observe_backend_event({"type": "response.output_audio.delta", "delta": _b64(8000)}) + + assert meter.snapshot().audio_seconds_received == 1.0 + + +def test_beta_flat_g711_format_bills_at_8000_bytes_per_second(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "session.update", "session": {"input_audio_format": "g711_ulaw"}}) + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(8000)}) + + assert meter.snapshot().audio_seconds_sent == 1.0 + + +def test_opus_format_contributes_no_audio_seconds(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event( + { + "type": "session.update", + "session": {"audio": {"input": {"format": {"type": "audio/opus"}}}}, + } + ) + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)}) + + assert meter.snapshot().audio_seconds_sent == 0.0 + + +def test_null_turn_detection_marks_push_to_talk_flat_and_nested(): + flat = XAIRealtimeUsageMeter() + flat.observe_client_event({"type": "session.update", "session": {"turn_detection": None}}) + assert flat.snapshot().push_to_talk is True + + nested = XAIRealtimeUsageMeter() + nested.observe_client_event( + { + "type": "session.update", + "session": {"audio": {"input": {"turn_detection": None}}}, + } + ) + assert nested.snapshot().push_to_talk is True + + +def test_client_null_type_turn_detection_marks_push_to_talk(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "session.update", "session": {"turn_detection": {"type": None}}}) + + assert meter.snapshot().push_to_talk is True + + +def test_backend_session_events_do_not_override_client_turn_detection(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}}) + meter.observe_backend_event( + {"type": "session.created", "session": {"turn_detection": {"type": None}}} + ) + meter.observe_backend_event({"type": "session.updated", "session": {"modalities": ["audio"]}}) + + assert meter.snapshot().push_to_talk is True + + +def test_later_client_server_vad_turn_detection_disables_push_to_talk(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}}) + meter.observe_client_event( + {"type": "session.update", "session": {"turn_detection": {"type": "server_vad"}}} + ) + + assert meter.snapshot().push_to_talk is False + + +def test_backend_turn_detection_alone_is_ignored(): + meter = XAIRealtimeUsageMeter() + + meter.observe_backend_event( + {"type": "session.created", "session": {"turn_detection": {"type": None}}} + ) + + assert meter.snapshot().push_to_talk is False + + +def test_server_vad_turn_detection_stays_not_push_to_talk(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "session.update", "session": {"turn_detection": {"type": "server_vad"}}}) + + assert meter.snapshot().push_to_talk is False + + +def test_non_string_audio_payloads_are_ignored(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": 123}) + meter.observe_backend_event({"type": "response.output_audio.delta", "delta": None}) + + snapshot = meter.snapshot() + assert snapshot.audio_seconds_sent == 0.0 + assert snapshot.audio_seconds_received == 0.0 + + +def test_padding_only_audio_never_subtracts_seconds(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)}) + for _ in range(10): + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": "=="}) + + assert meter.snapshot().audio_seconds_sent == 1.0 + + +def test_push_to_talk_after_content_keeps_server_vad_billing(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)}) + meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}}) + + assert meter.snapshot().push_to_talk is False + + +def test_server_vad_after_content_disables_push_to_talk(): + meter = XAIRealtimeUsageMeter() + + meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}}) + meter.observe_client_event({"type": "conversation.item.create", "item": {"type": "message", "content": []}}) + meter.observe_client_event({"type": "session.update", "session": {"turn_detection": {"type": "server_vad"}}}) + + assert meter.snapshot().push_to_talk is False diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index afbc73ab280..f58e9672c3b 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -14,6 +14,7 @@ from pydantic import BaseModel import litellm from litellm import TranscriptionResponse, model_cost +from litellm.constants import REALTIME_METERED_USAGE_KEY from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -27,6 +28,7 @@ from litellm.llms.fireworks_ai.cost_calculator import get_base_model_for_pricing from litellm.llms.together_ai.cost_calculator import get_model_params_and_category from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject @@ -946,6 +948,45 @@ def test_tiered_pricing_only_deployment_selects_router_model_id(): assert router_model_id in selected +def test_text_message_pricing_only_deployment_selects_router_model_id(): + """A deployment priced solely via ``input_cost_per_text_message`` (an xAI realtime + deployment with no per-second rate) counts as priced, so cost resolves against its + ``router_model_id`` entry instead of falling back to the stripped backend alias.""" + from litellm import Router + from litellm.cost_calculator import select_model_name_for_cost_calc + + router = Router( + model_list=[ + { + "model_name": "xai-text-only", + "litellm_params": { + "model": "xai/grok-voice-text-only-test", + "api_key": "sk-fake", + }, + "model_info": { + "input_cost_per_text_message": 0.004, + }, + }, + ] + ) + router_model_id = router.model_list[0]["model_info"]["id"] + + entry = litellm.model_cost[router_model_id] + assert entry.get("input_cost_per_token") is None + assert entry.get("input_cost_per_text_message") == 0.004 + assert litellm.model_cost["xai/grok-voice-text-only-test"].get("input_cost_per_text_message") is None + + selected = select_model_name_for_cost_calc( + model="xai/grok-voice-text-only-test", + completion_response=None, + custom_pricing=True, + custom_llm_provider="xai", + router_model_id=router_model_id, + ) + assert selected is not None + assert router_model_id in selected + + @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) def test_completion_cost_image_generation_reads_deployment_model_info_price_from_logging_metadata( _local_model_cost_map: None, metadata_key: str @@ -8802,11 +8843,12 @@ def _live_session_results(*events: dict[str, object]) -> OpenAIRealtimeStreamLis def _live_logging_object( start_time: datetime.datetime, end_time: datetime.datetime, + metered_usage: RealtimeMeteredUsage | None = None, ) -> Logging: - return cast( - Logging, - SimpleNamespace(model_call_details={"start_time": start_time, "end_time": end_time}), - ) + model_call_details: dict[str, object] = {"start_time": start_time, "end_time": end_time} + if metered_usage is not None: + model_call_details[REALTIME_METERED_USAGE_KEY] = metered_usage + return cast(Logging, SimpleNamespace(model_call_details=model_call_details)) def test_realtime_keeps_first_reported_session_model_for_pricing(_local_model_cost_map: None) -> None: @@ -8962,6 +9004,130 @@ def test_live_session_skips_non_finite_rate_for_next_candidate( assert cost == pytest.approx(4 * 0.25) +@pytest.fixture +def _xai_voice_test_model(_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "grok-voice-test", + { + "litellm_provider": "xai", + "mode": "realtime", + "input_cost_per_second": 0.0013333333333333333, + "input_cost_per_text_message": 0.004, + }, + ) + litellm.get_model_info.cache_clear() + + +def test_xai_realtime_server_vad_bills_wall_clock_audio(_xai_voice_test_model: None) -> None: + start_time: Final = datetime.datetime(2025, 1, 1) + + cost: Final = handle_realtime_stream_cost_calculation( + results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-test"}}), + combined_usage_object=Usage(), + custom_llm_provider="xai", + litellm_model_name="grok-voice-test", + litellm_logging_obj=_live_logging_object(start_time, start_time + datetime.timedelta(seconds=30)), + ) + + assert cost == pytest.approx(30 * 0.0013333333333333333) + + +def test_xai_realtime_push_to_talk_bills_only_metered_audio(_xai_voice_test_model: None) -> None: + start_time: Final = datetime.datetime(2025, 1, 1) + metered: Final = RealtimeMeteredUsage( + push_to_talk=True, audio_seconds_sent=2.0, audio_seconds_received=3.0, text_messages=0 + ) + + cost: Final = handle_realtime_stream_cost_calculation( + results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-test"}}), + combined_usage_object=Usage(), + custom_llm_provider="xai", + litellm_model_name="grok-voice-test", + litellm_logging_obj=_live_logging_object( + start_time, start_time + datetime.timedelta(seconds=600), metered_usage=metered + ), + ) + + assert cost == pytest.approx(5 * 0.0013333333333333333) + + +def test_xai_realtime_bills_text_messages(_xai_voice_test_model: None) -> None: + start_time: Final = datetime.datetime(2025, 1, 1) + metered: Final = RealtimeMeteredUsage( + push_to_talk=False, audio_seconds_sent=0.0, audio_seconds_received=0.0, text_messages=1 + ) + + cost: Final = handle_realtime_stream_cost_calculation( + results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-test"}}), + combined_usage_object=Usage(), + custom_llm_provider="xai", + litellm_model_name="grok-voice-test", + litellm_logging_obj=_live_logging_object( + start_time, start_time + datetime.timedelta(seconds=30), metered_usage=metered + ), + ) + + assert cost == pytest.approx(30 * 0.0013333333333333333 + 0.004) + + +def test_xai_realtime_missing_rates_bills_nothing( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "grok-voice-unpriced", + {"litellm_provider": "xai", "mode": "realtime"}, + ) + litellm.get_model_info.cache_clear() + start_time: Final = datetime.datetime(2025, 1, 1) + metered: Final = RealtimeMeteredUsage( + push_to_talk=True, audio_seconds_sent=2.0, audio_seconds_received=3.0, text_messages=2 + ) + + cost: Final = handle_realtime_stream_cost_calculation( + results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-unpriced"}}), + combined_usage_object=Usage(), + custom_llm_provider="xai", + litellm_model_name="grok-voice-unpriced", + litellm_logging_obj=_live_logging_object( + start_time, start_time + datetime.timedelta(seconds=30), metered_usage=metered + ), + ) + + assert cost == 0.0 + + +def test_non_xai_realtime_ignores_metered_usage(_local_model_cost_map: None) -> None: + start_time: Final = datetime.datetime(2025, 1, 1) + metered: Final = RealtimeMeteredUsage( + push_to_talk=True, audio_seconds_sent=2.0, audio_seconds_received=3.0, text_messages=5 + ) + results: Final = _live_session_results( + {"type": "session.started", "session": {"model": "gpt-live-1"}}, + {"type": "session.usage.updated", "usage": {"seconds": 15}}, + ) + + cost_with_metered: Final = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=Usage(), + custom_llm_provider="openai", + litellm_model_name="gpt-live-1", + litellm_logging_obj=_live_logging_object( + start_time, start_time + datetime.timedelta(seconds=20), metered_usage=metered + ), + ) + cost_without_metered: Final = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=Usage(), + custom_llm_provider="openai", + litellm_model_name="gpt-live-1", + litellm_logging_obj=_live_logging_object(start_time, start_time + datetime.timedelta(seconds=20)), + ) + + assert cost_with_metered == pytest.approx(cost_without_metered) + + def test_live_session_uses_wall_clock_duration_when_greater_than_reported(_local_model_cost_map: None) -> None: start_time: Final = datetime.datetime(2025, 1, 1) results: Final = _live_session_results( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b3eced92bb4..f50be89d032 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -814,6 +814,7 @@ def validate_model_cost_values(model_data, exceptions=None): "output_cost_per_pixel", "cost_per_second", "input_cost_per_second", + "input_cost_per_text_message", "output_cost_per_second", "output_cost_per_second_480p", "output_cost_per_second_720p", @@ -1020,6 +1021,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_request": {"type": "number"}, "cost_per_second": {"type": "number"}, "input_cost_per_second": {"type": "number"}, + "input_cost_per_text_message": {"type": "number"}, "input_cost_per_token": {"type": "number"}, "input_cost_per_token_above_128k_tokens": {"type": "number"}, "input_cost_per_audio_token_batches": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a8cf10363e4..53e07c3814a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35047,6 +35047,8 @@ export interface components { input_cost_per_query?: number | null; /** Input Cost Per Second */ input_cost_per_second?: number | null; + /** Input Cost Per Text Message */ + input_cost_per_text_message?: number | null; /** Input Cost Per Token */ input_cost_per_token?: number | null; /** Input Cost Per Token Above 100K Tokens */ @@ -50233,6 +50235,8 @@ export interface components { input_cost_per_query?: number | null; /** Input Cost Per Second */ input_cost_per_second?: number | null; + /** Input Cost Per Text Message */ + input_cost_per_text_message?: number | null; /** Input Cost Per Token */ input_cost_per_token?: number | null; /** Input Cost Per Token Above 100K Tokens */