mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-16 04:29:31 +00:00
feat(xai): bill grok voice realtime sessions (#46600)
* feat(xai): bill grok voice realtime sessions * chore: revert unrelated formatting churn in test_cost_calculator.py * refactor(xai): tidy realtime usage meter and tests * fix(xai): read push-to-talk mode from client session.update only * fix(xai): clamp decoded audio bytes and lock push-to-talk before content --------- Co-authored-by: kerry <kerry@berri.ai>
This commit is contained in:
Kerry Lu
GitHub
parent
0d741a9816
commit
efd8274315
15 changed files with 723 additions and 14 deletions
|
|
@ -397,6 +397,7 @@ MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolve
|
|||
MCP_PEEKED_BODY_SCOPE_KEY: Final = "litellm_mcp_peeked_body"
|
||||
MCP_ADMISSION_BODY_PEEK_TIMEOUT_SECONDS: Final = 5.0
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
|
||||
REALTIME_METERED_USAGE_KEY: Final = "realtime_metered_usage"
|
||||
OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS: Final = 30
|
||||
OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS: Final = 5
|
||||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.constants import (
|
|||
DEFAULT_MAX_LRU_CACHE_SIZE,
|
||||
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND,
|
||||
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
|
||||
|
|
@ -117,6 +118,7 @@ from litellm.types.llms.openai import (
|
|||
ResponseAPIUsage,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.llms.xai import RealtimeMeteredUsage
|
||||
from litellm.types.rerank import RerankBilledUnits, RerankResponse
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
|
|
@ -828,7 +830,14 @@ def _get_hidden_str_for_cost_calc(hidden_params: object, key: str) -> str | None
|
|||
|
||||
|
||||
_NON_TOKEN_RATE_FIELDS: Final = frozenset(
|
||||
{"cost_per_second", "input_cost_per_second", "output_cost_per_second", "input_cost_per_query", "tiered_pricing"}
|
||||
{
|
||||
"cost_per_second",
|
||||
"input_cost_per_second",
|
||||
"output_cost_per_second",
|
||||
"input_cost_per_query",
|
||||
"input_cost_per_text_message",
|
||||
"tiered_pricing",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -3119,31 +3128,43 @@ def _live_session_wall_clock_seconds(litellm_logging_obj: LitellmLoggingObject |
|
|||
return 0.0
|
||||
|
||||
|
||||
def _live_input_cost_per_second(model_name: str, custom_llm_provider: str) -> float | None:
|
||||
def _live_model_rate(
|
||||
model_name: str,
|
||||
custom_llm_provider: str,
|
||||
rate_field: Literal["input_cost_per_second", "input_cost_per_text_message"],
|
||||
) -> float | None:
|
||||
model_info: Final = _lookup_model_info_or_none(model_name, custom_llm_provider)
|
||||
if model_info is None:
|
||||
return None
|
||||
rate: Final = model_info.get("input_cost_per_second")
|
||||
rate: Final = model_info.get(rate_field)
|
||||
if isinstance(rate, (int, float)) and not isinstance(rate, bool) and math.isfinite(rate):
|
||||
return float(rate)
|
||||
return None
|
||||
|
||||
|
||||
def _first_live_input_cost_per_second(
|
||||
def _first_live_model_rate(
|
||||
potential_model_names: Sequence[str | None],
|
||||
custom_llm_provider: str,
|
||||
rate_field: Literal["input_cost_per_second", "input_cost_per_text_message"],
|
||||
) -> float:
|
||||
return next(
|
||||
(
|
||||
rate
|
||||
for model_name in potential_model_names
|
||||
if model_name is not None
|
||||
and (rate := _live_input_cost_per_second(model_name, custom_llm_provider)) is not None
|
||||
and (rate := _live_model_rate(model_name, custom_llm_provider, rate_field)) is not None
|
||||
),
|
||||
0.0,
|
||||
)
|
||||
|
||||
|
||||
def _xai_metered_usage(litellm_logging_obj: LitellmLoggingObject | None) -> RealtimeMeteredUsage | None:
|
||||
if litellm_logging_obj is None:
|
||||
return None
|
||||
usage: Final = litellm_logging_obj.model_call_details.get(REALTIME_METERED_USAGE_KEY)
|
||||
return usage if isinstance(usage, RealtimeMeteredUsage) else None
|
||||
|
||||
|
||||
def _live_response_token_costs(
|
||||
response: _LiveResponsePayload,
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -3303,9 +3324,28 @@ def handle_realtime_stream_cost_calculation(
|
|||
live_model_names: Final = (
|
||||
(custom_pricing_model, base_pricing_model, received_model, litellm_model_name) if live_event_seen else ()
|
||||
)
|
||||
live_voice_cost: Final = voice_seconds * _first_live_input_cost_per_second(
|
||||
live_voice_cost: Final = voice_seconds * _first_live_model_rate(
|
||||
live_model_names,
|
||||
custom_llm_provider,
|
||||
"input_cost_per_second",
|
||||
)
|
||||
xai_metered: Final = (
|
||||
_xai_metered_usage(litellm_logging_obj) if custom_llm_provider == LlmProviders.XAI.value else None
|
||||
)
|
||||
xai_audio_seconds: Final = (
|
||||
xai_metered.audio_seconds_sent + xai_metered.audio_seconds_received
|
||||
if xai_metered is not None and xai_metered.push_to_talk
|
||||
else _live_session_wall_clock_seconds(litellm_logging_obj)
|
||||
if custom_llm_provider == LlmProviders.XAI.value
|
||||
else 0.0
|
||||
)
|
||||
audio_duration_cost: Final = xai_audio_seconds * _first_live_model_rate(
|
||||
potential_model_names,
|
||||
custom_llm_provider,
|
||||
"input_cost_per_second",
|
||||
)
|
||||
text_message_cost: Final = (xai_metered.text_messages if xai_metered is not None else 0) * _first_live_model_rate(
|
||||
potential_model_names, custom_llm_provider, "input_cost_per_text_message"
|
||||
)
|
||||
live_response_costs: Final = tuple(
|
||||
_live_response_token_costs(
|
||||
|
|
@ -3319,10 +3359,17 @@ def handle_realtime_stream_cost_calculation(
|
|||
live_response_output_cost: Final = sum(costs[1] for costs in live_response_costs)
|
||||
input_cost: Final = input_cost_per_token + live_response_input_cost
|
||||
output_cost: Final = output_cost_per_token + live_response_output_cost
|
||||
total_cost: Final = input_cost + output_cost + transcription_cost + live_voice_cost
|
||||
total_cost: Final = (
|
||||
input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
|
||||
)
|
||||
additional_costs: Final = tuple(
|
||||
(cost_name, cost)
|
||||
for cost_name, cost in (("transcription_cost", transcription_cost), ("live_voice_cost", live_voice_cost))
|
||||
for cost_name, cost in (
|
||||
("transcription_cost", transcription_cost),
|
||||
("live_voice_cost", live_voice_cost),
|
||||
("audio_duration_cost", audio_duration_cost),
|
||||
("text_message_cost", text_message_cost),
|
||||
)
|
||||
if cost > 0
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,11 @@ from typing_extensions import ReadOnly
|
|||
|
||||
import litellm
|
||||
from litellm._logging import redact_internal_details_from_client_message, verbose_logger
|
||||
from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY
|
||||
from litellm.constants import (
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -114,6 +118,12 @@ class RealtimeEventNormalizer(Protocol):
|
|||
def patch_outgoing_session(self, session: dict) -> dict: ...
|
||||
|
||||
|
||||
class RealtimeUsageMeter(Protocol):
|
||||
def observe_client_event(self, event: Mapping[str, object]) -> None: ...
|
||||
def observe_backend_event(self, event: Mapping[str, object]) -> None: ...
|
||||
def snapshot(self) -> object: ...
|
||||
|
||||
|
||||
DefaultLoggedRealTimeEventTypes: Final = [
|
||||
"session.created",
|
||||
"response.create",
|
||||
|
|
@ -136,6 +146,7 @@ class RealTimeStreaming:
|
|||
backend_uses_beta_protocol: bool | None = None,
|
||||
force_transcription_model: str | None = None,
|
||||
event_normalizer: RealtimeEventNormalizer | None = None,
|
||||
usage_meter: RealtimeUsageMeter | None = None,
|
||||
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
|
||||
):
|
||||
self.websocket: _ClientWebSocket = websocket
|
||||
|
|
@ -198,6 +209,7 @@ class RealTimeStreaming:
|
|||
self._is_transcription_session: bool = force_transcription_model is not None
|
||||
# Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
|
||||
self._event_normalizer = event_normalizer
|
||||
self._usage_meter = usage_meter
|
||||
|
||||
@property
|
||||
def _session_created_sent_to_client(self) -> bool:
|
||||
|
|
@ -416,6 +428,8 @@ class RealTimeStreaming:
|
|||
async def log_messages(self):
|
||||
"""Log messages in list"""
|
||||
if self.logging_obj:
|
||||
if self._usage_meter is not None:
|
||||
self.logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] = self._usage_meter.snapshot()
|
||||
if self.input_messages:
|
||||
self.logging_obj.model_call_details["messages"] = self.input_messages
|
||||
if self.session_tools or self.tool_calls:
|
||||
|
|
@ -481,6 +495,10 @@ class RealTimeStreaming:
|
|||
sent = True
|
||||
return sent
|
||||
await self.backend_ws.send(message)
|
||||
if self._usage_meter is not None:
|
||||
sent_event: Final = self._parse_backend_event(message)
|
||||
if sent_event is not None:
|
||||
self._usage_meter.observe_client_event(sent_event)
|
||||
return True
|
||||
|
||||
def _enforce_transcription_session_model(self, message: str) -> str:
|
||||
|
|
@ -1113,6 +1131,9 @@ class RealTimeStreaming:
|
|||
if self._should_drop_event_from_client(event):
|
||||
continue
|
||||
|
||||
if self._usage_meter is not None:
|
||||
self._usage_meter.observe_backend_event(event)
|
||||
|
||||
if await self._handle_raw_backend_message(event, raw_response):
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from ....litellm_core_utils.realtime_errors import close_after_upstream_handshak
|
|||
from ....litellm_core_utils.realtime_streaming import (
|
||||
RealtimeEventNormalizer,
|
||||
RealTimeStreaming,
|
||||
RealtimeUsageMeter,
|
||||
client_sent_openai_beta_realtime_header,
|
||||
)
|
||||
from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
|
||||
|
|
@ -123,6 +124,10 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
"""
|
||||
return None
|
||||
|
||||
def _make_usage_meter(self) -> RealtimeUsageMeter | None:
|
||||
"""Return a per-session usage meter, or None when billing needs no metering."""
|
||||
return None
|
||||
|
||||
async def async_realtime(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -190,6 +195,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
model if (query_params or {}).get("intent") == "transcription" else None
|
||||
),
|
||||
event_normalizer=self._make_event_normalizer(),
|
||||
usage_meter=self._make_usage_meter(),
|
||||
)
|
||||
await realtime_streaming.bidirectional_forward()
|
||||
|
||||
|
|
|
|||
|
|
@ -8,9 +8,11 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
|
|||
"""
|
||||
|
||||
from litellm.constants import XAI_API_BASE
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealtimeUsageMeter
|
||||
|
||||
from ...openai.realtime.handler import OpenAIRealtime
|
||||
from .transformation import XAIRealtimeNormalizer
|
||||
from .usage_meter import XAIRealtimeUsageMeter
|
||||
|
||||
|
||||
class XAIRealtime(OpenAIRealtime):
|
||||
|
|
@ -33,6 +35,10 @@ class XAIRealtime(OpenAIRealtime):
|
|||
"""Return a fresh per-session XAI normalizer instance."""
|
||||
return XAIRealtimeNormalizer()
|
||||
|
||||
def _make_usage_meter(self) -> RealtimeUsageMeter:
|
||||
"""xAI bills realtime audio per second and text per client message."""
|
||||
return XAIRealtimeUsageMeter()
|
||||
|
||||
def _get_additional_headers(
|
||||
self,
|
||||
api_key: str,
|
||||
|
|
|
|||
163
litellm/llms/xai/realtime/usage_meter.py
Normal file
163
litellm/llms/xai/realtime/usage_meter.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""Counts billable xAI Grok Voice realtime usage from raw wire frames.
|
||||
|
||||
Audio format reference: https://docs.x.ai/voice-realtime.ws.json
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.types.llms.xai import RealtimeMeteredUsage
|
||||
|
||||
_PCM_DEFAULT_RATE: Final = 24000
|
||||
_G711_BYTES_PER_SECOND: Final = 8000
|
||||
_AUDIO_CONTENT_PART_TYPES: Final = frozenset({"input_audio", "audio"})
|
||||
_CONTENT_PART_LIST: Final = TypeAdapter(list[object])
|
||||
|
||||
|
||||
def _decoded_base64_length(value: object) -> int:
|
||||
"""Decoded byte length of a base64 string without decoding it."""
|
||||
if not isinstance(value, str):
|
||||
return 0
|
||||
padding: Final = min(len(value) - len(value.rstrip("=")), 2)
|
||||
return max(len(value) * 3 // 4 - padding, 0)
|
||||
|
||||
|
||||
def _audio_format_bytes_per_second(fmt: Mapping[str, object]) -> float:
|
||||
fmt_type: Final = fmt.get("type")
|
||||
if fmt_type is None or fmt_type == "audio/pcm":
|
||||
rate: Final = fmt.get("rate")
|
||||
rate_hz: Final = (
|
||||
int(rate)
|
||||
if isinstance(rate, (int, float)) and not isinstance(rate, bool) and rate > 0
|
||||
else _PCM_DEFAULT_RATE
|
||||
)
|
||||
return float(2 * rate_hz)
|
||||
if fmt_type in ("audio/pcmu", "audio/pcma"):
|
||||
return float(_G711_BYTES_PER_SECOND)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _beta_audio_format_bytes_per_second(name: object) -> float | None:
|
||||
if name == "pcm16":
|
||||
return float(2 * _PCM_DEFAULT_RATE)
|
||||
if name in ("g711_ulaw", "g711_alaw"):
|
||||
return float(_G711_BYTES_PER_SECOND)
|
||||
return None
|
||||
|
||||
|
||||
def _content_part_mappings(item: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
content: Final = item.get("content")
|
||||
raw_parts: Final = _CONTENT_PART_LIST.validate_python(content) if isinstance(content, list) else ()
|
||||
return tuple(part_map for part in raw_parts if (part_map := as_str_mapping(part)) is not None)
|
||||
|
||||
|
||||
def _is_billable_text_item(item: Mapping[str, object], parts: tuple[Mapping[str, object], ...]) -> bool:
|
||||
if item.get("type") == "function_call_output":
|
||||
return False
|
||||
return not any(part.get("type") in _AUDIO_CONTENT_PART_TYPES for part in parts)
|
||||
|
||||
|
||||
class XAIRealtimeUsageMeter:
|
||||
"""Counts billable xAI realtime usage from raw client and backend frames."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._push_to_talk: bool = False
|
||||
self._audio_seconds_sent: float = 0.0
|
||||
self._audio_seconds_received: float = 0.0
|
||||
self._text_messages: int = 0
|
||||
self._content_sent: bool = False
|
||||
self._input_bytes_per_second: float = float(2 * _PCM_DEFAULT_RATE)
|
||||
self._output_bytes_per_second: float = float(2 * _PCM_DEFAULT_RATE)
|
||||
|
||||
def observe_client_event(self, event: Mapping[str, object]) -> None:
|
||||
event_type: Final = event.get("type")
|
||||
if event_type == "session.update":
|
||||
session: Final = as_str_mapping(event.get("session"))
|
||||
if session is not None:
|
||||
self._apply_client_turn_detection(session)
|
||||
self._apply_session_config(session)
|
||||
elif event_type == "input_audio_buffer.append":
|
||||
self._content_sent = True
|
||||
if self._input_bytes_per_second > 0:
|
||||
self._audio_seconds_sent += _decoded_base64_length(event.get("audio")) / self._input_bytes_per_second
|
||||
elif event_type == "conversation.item.create":
|
||||
self._content_sent = True
|
||||
item: Final = as_str_mapping(event.get("item"))
|
||||
if item is not None:
|
||||
parts: Final = _content_part_mappings(item)
|
||||
if _is_billable_text_item(item, parts):
|
||||
self._text_messages += 1
|
||||
if self._input_bytes_per_second > 0:
|
||||
self._audio_seconds_sent += (
|
||||
sum(
|
||||
_decoded_base64_length(part.get("audio"))
|
||||
for part in parts
|
||||
if part.get("type") in _AUDIO_CONTENT_PART_TYPES
|
||||
)
|
||||
/ self._input_bytes_per_second
|
||||
)
|
||||
|
||||
def observe_backend_event(self, event: Mapping[str, object]) -> None:
|
||||
event_type: Final = event.get("type")
|
||||
if event_type in ("session.created", "session.updated"):
|
||||
session: Final = as_str_mapping(event.get("session"))
|
||||
if session is not None:
|
||||
self._apply_session_config(session)
|
||||
elif event_type in ("response.output_audio.delta", "response.audio.delta"):
|
||||
if self._output_bytes_per_second > 0:
|
||||
self._audio_seconds_received += (
|
||||
_decoded_base64_length(event.get("delta")) / self._output_bytes_per_second
|
||||
)
|
||||
|
||||
def snapshot(self) -> RealtimeMeteredUsage:
|
||||
return RealtimeMeteredUsage(
|
||||
push_to_talk=self._push_to_talk,
|
||||
audio_seconds_sent=self._audio_seconds_sent,
|
||||
audio_seconds_received=self._audio_seconds_received,
|
||||
text_messages=self._text_messages,
|
||||
)
|
||||
|
||||
def _apply_session_config(self, session: Mapping[str, object]) -> None:
|
||||
audio: Final = as_str_mapping(session.get("audio"))
|
||||
audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None
|
||||
|
||||
input_bps: Final = _beta_audio_format_bytes_per_second(session.get("input_audio_format"))
|
||||
if input_bps is not None:
|
||||
self._input_bytes_per_second = input_bps
|
||||
output_bps: Final = _beta_audio_format_bytes_per_second(session.get("output_audio_format"))
|
||||
if output_bps is not None:
|
||||
self._output_bytes_per_second = output_bps
|
||||
|
||||
if audio is None:
|
||||
return
|
||||
if audio_input is not None:
|
||||
input_format: Final = as_str_mapping(audio_input.get("format"))
|
||||
if input_format is not None:
|
||||
self._input_bytes_per_second = _audio_format_bytes_per_second(input_format)
|
||||
output_config: Final = as_str_mapping(audio.get("output"))
|
||||
if output_config is not None:
|
||||
output_format: Final = as_str_mapping(output_config.get("format"))
|
||||
if output_format is not None:
|
||||
self._output_bytes_per_second = _audio_format_bytes_per_second(output_format)
|
||||
|
||||
def _apply_client_turn_detection(self, session: Mapping[str, object]) -> None:
|
||||
audio: Final = as_str_mapping(session.get("audio"))
|
||||
audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None
|
||||
if "turn_detection" in session:
|
||||
self._apply_turn_detection(session["turn_detection"])
|
||||
elif audio_input is not None and "turn_detection" in audio_input:
|
||||
self._apply_turn_detection(audio_input["turn_detection"])
|
||||
|
||||
def _apply_turn_detection(self, value: object) -> None:
|
||||
config: Final = as_str_mapping(value)
|
||||
if value is not None and (config is None or "type" not in config):
|
||||
return
|
||||
push_to_talk: Final = value is None or (config is not None and config["type"] is None)
|
||||
# Switching to push-to-talk after content was sent keeps wall-clock
|
||||
# billing, so a late session.update cannot discount an elapsed VAD session
|
||||
if push_to_talk and self._content_sent:
|
||||
return
|
||||
self._push_to_talk = push_to_talk
|
||||
|
|
@ -1,6 +1,17 @@
|
|||
from dataclasses import dataclass
|
||||
from typing import Literal, TypedDict
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RealtimeMeteredUsage:
|
||||
"""Per-session xAI realtime usage counted from raw wire frames."""
|
||||
|
||||
push_to_talk: bool
|
||||
audio_seconds_sent: float
|
||||
audio_seconds_received: float
|
||||
text_messages: int
|
||||
|
||||
|
||||
class XAIWebSearchFilters(TypedDict, total=False):
|
||||
"""Filters for XAI web search tool"""
|
||||
|
||||
|
|
|
|||
|
|
@ -734,6 +734,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
|
|||
output_cost_per_token: float | None
|
||||
cost_per_second: ReadOnly[float | None]
|
||||
input_cost_per_second: float | None
|
||||
input_cost_per_text_message: ReadOnly[float | None]
|
||||
output_cost_per_second: float | None
|
||||
output_cost_per_second_480p: ReadOnly[float | None]
|
||||
output_cost_per_second_720p: ReadOnly[float | None]
|
||||
|
|
|
|||
|
|
@ -355,6 +355,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
input_cost_per_image_token_batches: ReadOnly[float | None]
|
||||
cost_per_second: ReadOnly[float | None]
|
||||
input_cost_per_second: float | None # for OpenAI Speech models
|
||||
input_cost_per_text_message: ReadOnly[float | None]
|
||||
input_cost_per_token_batches: float | None
|
||||
input_cost_per_video_token_batches: ReadOnly[float | None]
|
||||
input_cost_per_token_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
|
|
@ -2885,6 +2886,7 @@ class LoggedLiteLLMParams(TypedDict, total=False):
|
|||
no_log: bool | None
|
||||
cost_per_second: ReadOnly[float | None]
|
||||
input_cost_per_second: float | None
|
||||
input_cost_per_text_message: ReadOnly[float | None]
|
||||
input_cost_per_token: float | None
|
||||
output_cost_per_token: float | None
|
||||
output_cost_per_second: float | None
|
||||
|
|
@ -3857,6 +3859,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
## CUSTOM PRICING ##
|
||||
cost_per_second: float | None = None
|
||||
input_cost_per_second: float | None = None
|
||||
input_cost_per_text_message: float | None = None
|
||||
output_cost_per_second: float | None = None
|
||||
output_cost_per_second_1080p: float | None = None
|
||||
output_cost_per_second_480p: float | None = None
|
||||
|
|
|
|||
|
|
@ -6344,6 +6344,7 @@ def get_model_info_helper(
|
|||
input_cost_per_query=_model_info.get("input_cost_per_query", None),
|
||||
cost_per_second=_model_info.get("cost_per_second", None),
|
||||
input_cost_per_second=_model_info.get("input_cost_per_second", None),
|
||||
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
|
||||
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
|
||||
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
|
||||
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Coroutine, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -11,7 +12,11 @@ from websockets.exceptions import ConnectionClosed
|
|||
from websockets.frames import Close
|
||||
|
||||
import litellm
|
||||
from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY
|
||||
from litellm.constants import (
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.realtime_streaming import (
|
||||
RealTimeStreaming,
|
||||
|
|
@ -20,7 +25,9 @@ from litellm.litellm_core_utils.realtime_streaming import (
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer
|
||||
from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.llms.xai import RealtimeMeteredUsage
|
||||
from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeResponseTypedDict
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
|
|
@ -784,6 +791,60 @@ def test_collect_user_input_empty_transcript_not_collected():
|
|||
assert len(streaming.input_messages) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usage_meter_snapshot_stored_in_model_call_details():
|
||||
"""xAI metered usage from raw client/backend frames reaches the spend log."""
|
||||
client_ws = MagicMock()
|
||||
client_ws.send_text = AsyncMock()
|
||||
backend_ws = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(
|
||||
side_effect=[
|
||||
json.dumps(
|
||||
{"type": "session.created", "session": {"turn_detection": {"type": None}}}
|
||||
).encode(),
|
||||
json.dumps(
|
||||
{"type": "response.output_audio.delta", "delta": base64.b64encode(b"x" * 24000).decode()}
|
||||
).encode(),
|
||||
ConnectionClosed(None, None),
|
||||
]
|
||||
)
|
||||
logging_obj: Final = _mock_logging()
|
||||
logging_obj.model_call_details = {}
|
||||
logging_obj.async_success_handler = AsyncMock()
|
||||
logging_obj.success_handler = MagicMock()
|
||||
logging_obj.dispatch_success_handlers = AsyncMock()
|
||||
streaming = RealTimeStreaming(
|
||||
client_ws,
|
||||
backend_ws,
|
||||
logging_obj,
|
||||
event_normalizer=XAIRealtimeNormalizer(),
|
||||
usage_meter=XAIRealtimeUsageMeter(),
|
||||
)
|
||||
|
||||
await streaming._send_to_backend(json.dumps({"type": "session.update", "session": {"turn_detection": None}}))
|
||||
await streaming._send_to_backend(
|
||||
json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"y" * 48000).decode()})
|
||||
)
|
||||
await streaming._send_to_backend(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
}
|
||||
)
|
||||
)
|
||||
await streaming.backend_to_client_send_messages()
|
||||
await streaming.log_messages()
|
||||
|
||||
metered = logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY]
|
||||
assert isinstance(metered, RealtimeMeteredUsage)
|
||||
assert metered.push_to_talk is True
|
||||
assert metered.audio_seconds_sent == 1.0
|
||||
assert metered.audio_seconds_received == 0.5
|
||||
assert metered.text_messages == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_log_messages_sets_input_messages_on_logging_obj():
|
||||
"""
|
||||
|
|
|
|||
216
tests/unit/llms/xai/test_xai_realtime_usage_meter.py
Normal file
216
tests/unit/llms/xai/test_xai_realtime_usage_meter.py
Normal file
|
|
@ -0,0 +1,216 @@
|
|||
import base64
|
||||
|
||||
from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter
|
||||
|
||||
|
||||
def _b64(num_bytes: int) -> str:
|
||||
return base64.b64encode(b"x" * num_bytes).decode()
|
||||
|
||||
|
||||
def test_text_messages_counts_only_billable_items():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
}
|
||||
)
|
||||
meter.observe_client_event(
|
||||
{"type": "conversation.item.create", "item": {"type": "function_call_output", "output": "{}"}}
|
||||
)
|
||||
meter.observe_client_event(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {"type": "message", "role": "user", "content": [{"type": "input_audio", "audio": _b64(10)}]},
|
||||
}
|
||||
)
|
||||
meter.observe_client_event({"type": "response.create"})
|
||||
|
||||
assert meter.snapshot().text_messages == 1
|
||||
|
||||
|
||||
def test_input_audio_item_part_counts_as_sent_audio_not_text():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_audio", "audio": _b64(48000)}],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
snapshot = meter.snapshot()
|
||||
assert snapshot.audio_seconds_sent == 1.0
|
||||
assert snapshot.text_messages == 0
|
||||
|
||||
|
||||
def test_audio_seconds_default_pcm24():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)})
|
||||
meter.observe_backend_event({"type": "response.output_audio.delta", "delta": _b64(24000)})
|
||||
meter.observe_backend_event({"type": "response.audio.delta", "delta": _b64(24000)})
|
||||
|
||||
snapshot = meter.snapshot()
|
||||
assert snapshot.audio_seconds_sent == 1.0
|
||||
assert snapshot.audio_seconds_received == 1.0
|
||||
|
||||
|
||||
def test_session_update_nested_pcm_rate_changes_seconds():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}},
|
||||
}
|
||||
)
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(32000)})
|
||||
|
||||
assert meter.snapshot().audio_seconds_sent == 1.0
|
||||
|
||||
|
||||
def test_g711_format_bills_at_8000_bytes_per_second():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_backend_event(
|
||||
{
|
||||
"type": "session.created",
|
||||
"session": {"audio": {"output": {"format": {"type": "audio/pcmu"}}}},
|
||||
}
|
||||
)
|
||||
meter.observe_backend_event({"type": "response.output_audio.delta", "delta": _b64(8000)})
|
||||
|
||||
assert meter.snapshot().audio_seconds_received == 1.0
|
||||
|
||||
|
||||
def test_beta_flat_g711_format_bills_at_8000_bytes_per_second():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "session.update", "session": {"input_audio_format": "g711_ulaw"}})
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(8000)})
|
||||
|
||||
assert meter.snapshot().audio_seconds_sent == 1.0
|
||||
|
||||
|
||||
def test_opus_format_contributes_no_audio_seconds():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {"audio": {"input": {"format": {"type": "audio/opus"}}}},
|
||||
}
|
||||
)
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)})
|
||||
|
||||
assert meter.snapshot().audio_seconds_sent == 0.0
|
||||
|
||||
|
||||
def test_null_turn_detection_marks_push_to_talk_flat_and_nested():
|
||||
flat = XAIRealtimeUsageMeter()
|
||||
flat.observe_client_event({"type": "session.update", "session": {"turn_detection": None}})
|
||||
assert flat.snapshot().push_to_talk is True
|
||||
|
||||
nested = XAIRealtimeUsageMeter()
|
||||
nested.observe_client_event(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {"audio": {"input": {"turn_detection": None}}},
|
||||
}
|
||||
)
|
||||
assert nested.snapshot().push_to_talk is True
|
||||
|
||||
|
||||
def test_client_null_type_turn_detection_marks_push_to_talk():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "session.update", "session": {"turn_detection": {"type": None}}})
|
||||
|
||||
assert meter.snapshot().push_to_talk is True
|
||||
|
||||
|
||||
def test_backend_session_events_do_not_override_client_turn_detection():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}})
|
||||
meter.observe_backend_event(
|
||||
{"type": "session.created", "session": {"turn_detection": {"type": None}}}
|
||||
)
|
||||
meter.observe_backend_event({"type": "session.updated", "session": {"modalities": ["audio"]}})
|
||||
|
||||
assert meter.snapshot().push_to_talk is True
|
||||
|
||||
|
||||
def test_later_client_server_vad_turn_detection_disables_push_to_talk():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}})
|
||||
meter.observe_client_event(
|
||||
{"type": "session.update", "session": {"turn_detection": {"type": "server_vad"}}}
|
||||
)
|
||||
|
||||
assert meter.snapshot().push_to_talk is False
|
||||
|
||||
|
||||
def test_backend_turn_detection_alone_is_ignored():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_backend_event(
|
||||
{"type": "session.created", "session": {"turn_detection": {"type": None}}}
|
||||
)
|
||||
|
||||
assert meter.snapshot().push_to_talk is False
|
||||
|
||||
|
||||
def test_server_vad_turn_detection_stays_not_push_to_talk():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "session.update", "session": {"turn_detection": {"type": "server_vad"}}})
|
||||
|
||||
assert meter.snapshot().push_to_talk is False
|
||||
|
||||
|
||||
def test_non_string_audio_payloads_are_ignored():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": 123})
|
||||
meter.observe_backend_event({"type": "response.output_audio.delta", "delta": None})
|
||||
|
||||
snapshot = meter.snapshot()
|
||||
assert snapshot.audio_seconds_sent == 0.0
|
||||
assert snapshot.audio_seconds_received == 0.0
|
||||
|
||||
|
||||
def test_padding_only_audio_never_subtracts_seconds():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)})
|
||||
for _ in range(10):
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": "=="})
|
||||
|
||||
assert meter.snapshot().audio_seconds_sent == 1.0
|
||||
|
||||
|
||||
def test_push_to_talk_after_content_keeps_server_vad_billing():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "input_audio_buffer.append", "audio": _b64(48000)})
|
||||
meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}})
|
||||
|
||||
assert meter.snapshot().push_to_talk is False
|
||||
|
||||
|
||||
def test_server_vad_after_content_disables_push_to_talk():
|
||||
meter = XAIRealtimeUsageMeter()
|
||||
|
||||
meter.observe_client_event({"type": "session.update", "session": {"turn_detection": None}})
|
||||
meter.observe_client_event({"type": "conversation.item.create", "item": {"type": "message", "content": []}})
|
||||
meter.observe_client_event({"type": "session.update", "session": {"turn_detection": {"type": "server_vad"}}})
|
||||
|
||||
assert meter.snapshot().push_to_talk is False
|
||||
|
|
@ -14,6 +14,7 @@ from pydantic import BaseModel
|
|||
|
||||
import litellm
|
||||
from litellm import TranscriptionResponse, model_cost
|
||||
from litellm.constants import REALTIME_METERED_USAGE_KEY
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
|
@ -27,6 +28,7 @@ from litellm.llms.fireworks_ai.cost_calculator import get_base_model_for_pricing
|
|||
from litellm.llms.together_ai.cost_calculator import get_model_params_and_category
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse
|
||||
from litellm.types.llms.xai import RealtimeMeteredUsage
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage
|
||||
from litellm.types.videos.main import VideoObject
|
||||
|
|
@ -946,6 +948,45 @@ def test_tiered_pricing_only_deployment_selects_router_model_id():
|
|||
assert router_model_id in selected
|
||||
|
||||
|
||||
def test_text_message_pricing_only_deployment_selects_router_model_id():
|
||||
"""A deployment priced solely via ``input_cost_per_text_message`` (an xAI realtime
|
||||
deployment with no per-second rate) counts as priced, so cost resolves against its
|
||||
``router_model_id`` entry instead of falling back to the stripped backend alias."""
|
||||
from litellm import Router
|
||||
from litellm.cost_calculator import select_model_name_for_cost_calc
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "xai-text-only",
|
||||
"litellm_params": {
|
||||
"model": "xai/grok-voice-text-only-test",
|
||||
"api_key": "sk-fake",
|
||||
},
|
||||
"model_info": {
|
||||
"input_cost_per_text_message": 0.004,
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
router_model_id = router.model_list[0]["model_info"]["id"]
|
||||
|
||||
entry = litellm.model_cost[router_model_id]
|
||||
assert entry.get("input_cost_per_token") is None
|
||||
assert entry.get("input_cost_per_text_message") == 0.004
|
||||
assert litellm.model_cost["xai/grok-voice-text-only-test"].get("input_cost_per_text_message") is None
|
||||
|
||||
selected = select_model_name_for_cost_calc(
|
||||
model="xai/grok-voice-text-only-test",
|
||||
completion_response=None,
|
||||
custom_pricing=True,
|
||||
custom_llm_provider="xai",
|
||||
router_model_id=router_model_id,
|
||||
)
|
||||
assert selected is not None
|
||||
assert router_model_id in selected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"])
|
||||
def test_completion_cost_image_generation_reads_deployment_model_info_price_from_logging_metadata(
|
||||
_local_model_cost_map: None, metadata_key: str
|
||||
|
|
@ -8802,11 +8843,12 @@ def _live_session_results(*events: dict[str, object]) -> OpenAIRealtimeStreamLis
|
|||
def _live_logging_object(
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
metered_usage: RealtimeMeteredUsage | None = None,
|
||||
) -> Logging:
|
||||
return cast(
|
||||
Logging,
|
||||
SimpleNamespace(model_call_details={"start_time": start_time, "end_time": end_time}),
|
||||
)
|
||||
model_call_details: dict[str, object] = {"start_time": start_time, "end_time": end_time}
|
||||
if metered_usage is not None:
|
||||
model_call_details[REALTIME_METERED_USAGE_KEY] = metered_usage
|
||||
return cast(Logging, SimpleNamespace(model_call_details=model_call_details))
|
||||
|
||||
|
||||
def test_realtime_keeps_first_reported_session_model_for_pricing(_local_model_cost_map: None) -> None:
|
||||
|
|
@ -8962,6 +9004,130 @@ def test_live_session_skips_non_finite_rate_for_next_candidate(
|
|||
assert cost == pytest.approx(4 * 0.25)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _xai_voice_test_model(_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"grok-voice-test",
|
||||
{
|
||||
"litellm_provider": "xai",
|
||||
"mode": "realtime",
|
||||
"input_cost_per_second": 0.0013333333333333333,
|
||||
"input_cost_per_text_message": 0.004,
|
||||
},
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_xai_realtime_server_vad_bills_wall_clock_audio(_xai_voice_test_model: None) -> None:
|
||||
start_time: Final = datetime.datetime(2025, 1, 1)
|
||||
|
||||
cost: Final = handle_realtime_stream_cost_calculation(
|
||||
results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-test"}}),
|
||||
combined_usage_object=Usage(),
|
||||
custom_llm_provider="xai",
|
||||
litellm_model_name="grok-voice-test",
|
||||
litellm_logging_obj=_live_logging_object(start_time, start_time + datetime.timedelta(seconds=30)),
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(30 * 0.0013333333333333333)
|
||||
|
||||
|
||||
def test_xai_realtime_push_to_talk_bills_only_metered_audio(_xai_voice_test_model: None) -> None:
|
||||
start_time: Final = datetime.datetime(2025, 1, 1)
|
||||
metered: Final = RealtimeMeteredUsage(
|
||||
push_to_talk=True, audio_seconds_sent=2.0, audio_seconds_received=3.0, text_messages=0
|
||||
)
|
||||
|
||||
cost: Final = handle_realtime_stream_cost_calculation(
|
||||
results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-test"}}),
|
||||
combined_usage_object=Usage(),
|
||||
custom_llm_provider="xai",
|
||||
litellm_model_name="grok-voice-test",
|
||||
litellm_logging_obj=_live_logging_object(
|
||||
start_time, start_time + datetime.timedelta(seconds=600), metered_usage=metered
|
||||
),
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(5 * 0.0013333333333333333)
|
||||
|
||||
|
||||
def test_xai_realtime_bills_text_messages(_xai_voice_test_model: None) -> None:
|
||||
start_time: Final = datetime.datetime(2025, 1, 1)
|
||||
metered: Final = RealtimeMeteredUsage(
|
||||
push_to_talk=False, audio_seconds_sent=0.0, audio_seconds_received=0.0, text_messages=1
|
||||
)
|
||||
|
||||
cost: Final = handle_realtime_stream_cost_calculation(
|
||||
results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-test"}}),
|
||||
combined_usage_object=Usage(),
|
||||
custom_llm_provider="xai",
|
||||
litellm_model_name="grok-voice-test",
|
||||
litellm_logging_obj=_live_logging_object(
|
||||
start_time, start_time + datetime.timedelta(seconds=30), metered_usage=metered
|
||||
),
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(30 * 0.0013333333333333333 + 0.004)
|
||||
|
||||
|
||||
def test_xai_realtime_missing_rates_bills_nothing(
|
||||
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"grok-voice-unpriced",
|
||||
{"litellm_provider": "xai", "mode": "realtime"},
|
||||
)
|
||||
litellm.get_model_info.cache_clear()
|
||||
start_time: Final = datetime.datetime(2025, 1, 1)
|
||||
metered: Final = RealtimeMeteredUsage(
|
||||
push_to_talk=True, audio_seconds_sent=2.0, audio_seconds_received=3.0, text_messages=2
|
||||
)
|
||||
|
||||
cost: Final = handle_realtime_stream_cost_calculation(
|
||||
results=_live_session_results({"type": "session.created", "session": {"model": "grok-voice-unpriced"}}),
|
||||
combined_usage_object=Usage(),
|
||||
custom_llm_provider="xai",
|
||||
litellm_model_name="grok-voice-unpriced",
|
||||
litellm_logging_obj=_live_logging_object(
|
||||
start_time, start_time + datetime.timedelta(seconds=30), metered_usage=metered
|
||||
),
|
||||
)
|
||||
|
||||
assert cost == 0.0
|
||||
|
||||
|
||||
def test_non_xai_realtime_ignores_metered_usage(_local_model_cost_map: None) -> None:
|
||||
start_time: Final = datetime.datetime(2025, 1, 1)
|
||||
metered: Final = RealtimeMeteredUsage(
|
||||
push_to_talk=True, audio_seconds_sent=2.0, audio_seconds_received=3.0, text_messages=5
|
||||
)
|
||||
results: Final = _live_session_results(
|
||||
{"type": "session.started", "session": {"model": "gpt-live-1"}},
|
||||
{"type": "session.usage.updated", "usage": {"seconds": 15}},
|
||||
)
|
||||
|
||||
cost_with_metered: Final = handle_realtime_stream_cost_calculation(
|
||||
results=results,
|
||||
combined_usage_object=Usage(),
|
||||
custom_llm_provider="openai",
|
||||
litellm_model_name="gpt-live-1",
|
||||
litellm_logging_obj=_live_logging_object(
|
||||
start_time, start_time + datetime.timedelta(seconds=20), metered_usage=metered
|
||||
),
|
||||
)
|
||||
cost_without_metered: Final = handle_realtime_stream_cost_calculation(
|
||||
results=results,
|
||||
combined_usage_object=Usage(),
|
||||
custom_llm_provider="openai",
|
||||
litellm_model_name="gpt-live-1",
|
||||
litellm_logging_obj=_live_logging_object(start_time, start_time + datetime.timedelta(seconds=20)),
|
||||
)
|
||||
|
||||
assert cost_with_metered == pytest.approx(cost_without_metered)
|
||||
|
||||
|
||||
def test_live_session_uses_wall_clock_duration_when_greater_than_reported(_local_model_cost_map: None) -> None:
|
||||
start_time: Final = datetime.datetime(2025, 1, 1)
|
||||
results: Final = _live_session_results(
|
||||
|
|
|
|||
|
|
@ -814,6 +814,7 @@ def validate_model_cost_values(model_data, exceptions=None):
|
|||
"output_cost_per_pixel",
|
||||
"cost_per_second",
|
||||
"input_cost_per_second",
|
||||
"input_cost_per_text_message",
|
||||
"output_cost_per_second",
|
||||
"output_cost_per_second_480p",
|
||||
"output_cost_per_second_720p",
|
||||
|
|
@ -1020,6 +1021,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"input_cost_per_request": {"type": "number"},
|
||||
"cost_per_second": {"type": "number"},
|
||||
"input_cost_per_second": {"type": "number"},
|
||||
"input_cost_per_text_message": {"type": "number"},
|
||||
"input_cost_per_token": {"type": "number"},
|
||||
"input_cost_per_token_above_128k_tokens": {"type": "number"},
|
||||
"input_cost_per_audio_token_batches": {"type": "number"},
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -35047,6 +35047,8 @@ export interface components {
|
|||
input_cost_per_query?: number | null;
|
||||
/** Input Cost Per Second */
|
||||
input_cost_per_second?: number | null;
|
||||
/** Input Cost Per Text Message */
|
||||
input_cost_per_text_message?: number | null;
|
||||
/** Input Cost Per Token */
|
||||
input_cost_per_token?: number | null;
|
||||
/** Input Cost Per Token Above 100K Tokens */
|
||||
|
|
@ -50233,6 +50235,8 @@ export interface components {
|
|||
input_cost_per_query?: number | null;
|
||||
/** Input Cost Per Second */
|
||||
input_cost_per_second?: number | null;
|
||||
/** Input Cost Per Text Message */
|
||||
input_cost_per_text_message?: number | null;
|
||||
/** Input Cost Per Token */
|
||||
input_cost_per_token?: number | null;
|
||||
/** Input Cost Per Token Above 100K Tokens */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue