scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

chore: merge main into litellm_e2e_codex_mcp_tools

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-11-03 09:07:26 -08:00
commit 89020f578f
158 changed files with 19116 additions and 1177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-unit.yml vendored
View file

@ -441,6 +441,7 @@ jobs:
tests/unit/ocr
tests/unit/passthrough
tests/unit/rag
tests/unit/cookbook
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search

245
cookbook/gpt_realtime_translate.py Normal file
View file

@ -0,0 +1,245 @@
#!/usr/bin/env python3
import argparse
import asyncio
import base64
import json
import os
import sys
import wave
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlencode, urlsplit, urlunsplit
import websockets
from websockets.asyncio.client import ClientConnection
SAMPLE_RATE = 24_000
CHANNELS = 1
SAMPLE_WIDTH = 2
CHUNK_DURATION_SECONDS = 0.1
CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS)
RESPONSE_TIMEOUT_SECONDS = 30.0
AUDIO_EVENT_TYPES = frozenset(
{
"session.output_audio.delta",
"response.audio.delta",
"response.output_audio.delta",
}
)
TRANSCRIPT_EVENT_TYPES = frozenset(
{
"session.output_transcript.delta",
"response.text.delta",
"response.output_audio_transcript.delta",
}
)
@dataclass(frozen=True, slots=True)
class Settings:
input_wav: Path
output_wav: Path
base_url: str
model: str
target_language: str
trailing_silence_seconds: float
api_key: str
def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None:
sys.stdout.write(f"{message}{end}")
if flush:
sys.stdout.flush()
def write_stderr(message: str) -> None:
sys.stderr.write(f"{message}\n")
def parse_args(argv: Sequence[str] | None = None) -> Settings | str:
parser = argparse.ArgumentParser(
description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio",
)
parser.add_argument("input_wav", type=Path)
parser.add_argument("--output", type=Path, default=Path("translated.wav"))
parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000"))
parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate"))
parser.add_argument("--target-language", default="fr")
parser.add_argument("--trailing-silence", type=float, default=1.5)
parsed = parser.parse_args(argv)
api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY")
if not api_key:
return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script"
if parsed.trailing_silence < 0:
return "--trailing-silence must be zero or greater"
return Settings(
input_wav=parsed.input_wav,
output_wav=parsed.output,
base_url=parsed.base_url,
model=parsed.model,
target_language=parsed.target_language,
trailing_silence_seconds=parsed.trailing_silence,
api_key=api_key,
)
def translation_url(base_url: str, model: str) -> str | None:
parsed = urlsplit(base_url.rstrip("/"))
scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme)
if not scheme or not parsed.netloc:
return None
base_path = parsed.path.rstrip("/")
realtime_path = (
f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations"
)
return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), ""))
def read_pcm16_wav(path: Path) -> bytes | str:
try:
with wave.open(str(path), "rb") as source:
actual_format = (
source.getnchannels(),
source.getsampwidth(),
source.getframerate(),
source.getcomptype(),
)
expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE")
if actual_format != expected_format:
return (
f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received "
f"channels={actual_format[0]}, sample_width={actual_format[1]}, "
f"sample_rate={actual_format[2]}, compression={actual_format[3]}"
)
return source.readframes(source.getnframes())
except (OSError, EOFError, wave.Error) as exc:
return f"Unable to read {path}: {exc}"
def audio_chunks(audio: bytes) -> Iterator[bytes]:
return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES))
def audio_message(audio: bytes) -> str:
return json.dumps(
{
"type": "session.input_audio_buffer.append",
"audio": base64.b64encode(audio).decode("ascii"),
}
)
async def configure_session(connection: ClientConnection, target_language: str) -> str | None:
await connection.send(
json.dumps(
{
"type": "session.update",
"session": {"audio": {"output": {"language": target_language}}},
}
)
)
while True:
raw_event = await asyncio.wait_for(connection.recv(), timeout=20)
event = json.loads(raw_event)
event_type = event.get("type")
if event_type == "session.created":
write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}")
if event_type == "session.updated":
return None
if event_type == "error":
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
async def send_audio(
connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event
) -> None:
silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
try:
for chunk in audio_chunks(pcm + silence):
await connection.send(audio_message(chunk))
await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH))
await connection.send(json.dumps({"type": "session.close"}))
finally:
finished.set()
async def receive_translation(
connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event
) -> str | None:
audio_received = asyncio.Event()
try:
with wave.open(str(output_path), "wb") as output:
output.setnchannels(CHANNELS)
output.setsampwidth(SAMPLE_WIDTH)
output.setframerate(SAMPLE_RATE)
write_stdout("Translation: ", end="", flush=True)
while True:
try:
raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS)
except TimeoutError:
if not sender_finished.is_set():
continue
return (
f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session"
)
event = json.loads(raw_event)
event_type = event.get("type")
if event_type in AUDIO_EVENT_TYPES:
output.writeframes(base64.b64decode(event.get("delta", ""), validate=True))
audio_received.set()
elif event_type in TRANSCRIPT_EVENT_TYPES:
write_stdout(event.get("delta", event.get("text", "")), end="", flush=True)
elif event_type == "error":
return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}"
elif event_type == "session.closed":
write_stdout()
return None if audio_received.is_set() else "The translation stream ended without translated audio"
except (OSError, wave.Error) as exc:
return f"Unable to write {output_path}: {exc}"
async def translate(settings: Settings, pcm: bytes) -> str | None:
url = translation_url(settings.base_url, settings.model)
if not url:
return f"Invalid --base-url: {settings.base_url}"
sender_finished = asyncio.Event()
try:
async with websockets.connect(
url,
additional_headers={"Authorization": f"Bearer {settings.api_key}"},
proxy=None,
open_timeout=20,
close_timeout=5,
) as connection:
configuration_error = await configure_session(connection, settings.target_language)
if configuration_error:
return configuration_error
async with asyncio.TaskGroup() as tasks:
receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished))
tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished))
return receiver.result()
except Exception as exc:
return f"Translation failed: {type(exc).__name__}: {exc}"
def main(argv: Sequence[str] | None = None) -> int:
settings = parse_args(argv)
if isinstance(settings, str):
write_stderr(settings)
return 2
pcm = read_pcm16_wav(settings.input_wav)
if isinstance(pcm, str):
write_stderr(pcm)
return 2
error = asyncio.run(translate(settings, pcm))
if error:
write_stderr(error)
return 1
write_stdout(f"Translated audio: {settings.output_wav}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

6
litellm-rust/crates/model-catalog/src/vocabulary.rs
View file

@ -93,6 +93,12 @@ pub enum Endpoint {
Realtime,
#[serde(rename = "/v1/realtime/transcription_sessions")]
RealtimeTranscriptionSessions,
#[serde(rename = "/v1/realtime/translations")]
RealtimeTranslations,
#[serde(rename = "/v1/realtime/translations/calls")]
RealtimeTranslationCalls,
#[serde(rename = "/v1/realtime/translations/client_secrets")]
RealtimeTranslationClientSecrets,
#[serde(rename = "/v1/videos")]
Videos,
#[serde(rename = "/v1/ocr")]

16
litellm/__init__.py
View file

@ -8,7 +8,7 @@ _apply_process_defaults()
import threading
import os
from collections.abc import Mapping, Sequence
from collections.abc import Sequence
from typing import (
Any,
Callable,
@ -1562,7 +1562,9 @@ from .realtime_api.main import (
_arealtime,
acreate_realtime_client_secret,
acreate_realtime_transcription_session,
acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret,
arealtime_calls,
arealtime_translation_calls as arealtime_translation_calls,
)
from .responses.main import _aresponses_websocket
@ -1811,6 +1813,9 @@ if TYPE_CHECKING:
from .llms.vertex_ai.rerank.transformation import (
VertexAIRerankConfig as VertexAIRerankConfig,
)
from .llms.together_ai.chat.transformation import (
TogetherAIChatConfig as TogetherAIChatConfig,
)
from .llms.fireworks_ai.rerank.transformation import (
FireworksAIRerankConfig as FireworksAIRerankConfig,
)
@ -1856,9 +1861,6 @@ if TYPE_CHECKING:
BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig,
)
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
from .llms.together_ai.chat.transformation import (
TogetherAIChatConfig as TogetherAIChatConfig,
)
from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig as VertexGeminiConfig,
@ -2017,6 +2019,9 @@ if TYPE_CHECKING:
from .llms.xai.responses.transformation import (
XAIResponsesAPIConfig as XAIResponsesAPIConfig,
)
from .llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
)
from .llms.litellm_proxy.responses.transformation import (
LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
)
@ -2044,9 +2049,6 @@ if TYPE_CHECKING:
from .llms.gemini.interactions.transformation import (
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
)
from .llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
)
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIOSeriesConfig,
OpenAIOSeriesConfig as OpenAIO1Config,

22
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -11,7 +11,7 @@ from typing import (
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
)
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponsesAPIResponse
@ -20,15 +20,15 @@ if TYPE_CHECKING:
class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
model: str
messages: list
optional_params: dict
litellm_params: dict
headers: dict
model_response: "ModelResponse"
logging_obj: "LiteLLMLoggingObj"
custom_llm_provider: str
encoding: object
model: ReadOnly[str]
messages: ReadOnly[list]
optional_params: ReadOnly[dict[str, object]]
litellm_params: ReadOnly[dict[str, object]]
headers: ReadOnly[dict]
model_response: ReadOnly["ModelResponse"]
logging_obj: ReadOnly["LiteLLMLoggingObj"]
custom_llm_provider: ReadOnly[str]
encoding: ReadOnly[object]
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
request_data: Final = self.transformation_handler.transform_request(
model=model,
@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
try:
request_data: Final = self.transformation_handler.transform_request(

14
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -34,7 +34,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
replay_items,
requested_phase,
)
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_chat_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
@ -1166,10 +1166,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
optional_params = self._extract_extra_body_params(optional_params)
provider: Final = _served_provider(litellm_params)
base_model: Final = litellm_params.get("base_model")
supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model, provider) or (
isinstance(base_model, str)
and bool(base_model)
and supports_openai_prompt_cache_breakpoint(base_model, provider)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,)
supports_prompt_cache_breakpoint: Final = any(
supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models
)
converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
messages,
@ -1235,10 +1235,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
api_model: Final = model
from litellm.types.utils import CallTypes
setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params)
request_data: Final = {

4
litellm/constants.py
View file

@ -451,6 +451,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
{"response.completed", "response.incomplete", "response.failed"}
)
REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally"
# SSL/TLS cipher configuration for faster handshakes
# Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones
@ -971,7 +972,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [
OPENAI_TRANSCRIPTION_PARAMS: Final = [
"language",
"languages",
"keywords",
"response_format",
"stream",
"timestamp_granularities",
]

212
litellm/cost_calculator.py
View file

@ -23,7 +23,8 @@ from litellm.constants import (
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
REALTIME_METERED_USAGE_KEY,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.decisions_pricing import (
decisions_pricing_fields,
@ -324,12 +325,41 @@ def _get_additional_costs(
return None
def _transcription_usage_has_token_details(
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second",
"output_cost_per_second",
"input_cost_per_audio_token",
"output_cost_per_audio_token",
"input_cost_per_token",
"output_cost_per_token",
]
_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
"input_cost_per_token",
"output_cost_per_token",
"input_cost_per_audio_token",
"output_cost_per_audio_token",
)
_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = (
"input_cost_per_second",
"output_cost_per_second",
)
def _transcription_uses_token_pricing(
model: str,
custom_llm_provider: str | None,
usage_block: Usage | None,
) -> bool:
if usage_block is None:
return False
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
if (
_declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None
and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None
):
return False
prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
@ -343,13 +373,38 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool:
if pricing_model is None or custom_llm_provider is None:
return False
transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider)
transcription_metadata: Final = (
as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None
)
return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1
def _transcription_duration_for_cost(
completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None
) -> float:
hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {}
measured_duration: Final = normalized_audio_duration_seconds(
hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration"))
)
if measured_duration is not None and measured_duration > 0:
return measured_duration
provider_duration: Final = _get_transcription_usage_duration(completion_response)
if (
provider_duration is not None
and provider_duration > 0
and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model))
):
return provider_duration
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return 0.0
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -701,7 +756,7 @@ def cost_per_token(
vertex_location=vertex_location,
)
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block):
return generic_cost_per_token(
model=model,
usage=usage_block,
@ -1283,6 +1338,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage:
)
def _get_transcription_usage_duration(completion_response: object) -> float | None:
response_fields: Final = as_str_mapping(completion_response)
usage_object: Final = (
response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage")
)
usage_fields: Final = as_str_mapping(usage_object)
usage_type: Final = (
usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type")
)
if usage_type != "duration":
return None
seconds: Final = (
usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds")
)
return normalized_audio_duration_seconds(seconds)
def _extraction_usage(usage: ExtractionUsage) -> Usage:
return Usage(
prompt_tokens=usage.input_tokens,
@ -2008,20 +2080,8 @@ def completion_cost(
prompt,
)
elif call_type in _TRANSCRIPTION_CALL_TYPES:
# Check _hidden_params first (duration stored there to
# avoid polluting the response body), then fall back to
# the response attribute (for verbose_json responses that
# naturally include duration from the provider).
_hidden = getattr(completion_response, "_hidden_params", {}) or {}
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
audio_transcription_file_duration = _transcription_duration_for_cost(
completion_response, model, requested_model, custom_llm_provider
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):
@ -2131,6 +2191,7 @@ def completion_cost(
litellm_logging_obj=litellm_logging_obj,
custom_pricing_model=selected_model if custom_pricing else None,
base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None),
deployment_models=(base_model, requested_model),
)
elif call_type == _MCP_CALL_TYPE:
from litellm.proxy.mcp_server.cost_calculator import (
@ -3409,8 +3470,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
def create_logging_realtime_object(
usage: Usage, results: OpenAIRealtimeStreamList
) -> LiteLLMRealtimeStreamLoggingObject:
translated_input_seconds, _ = _translation_session_seconds(results)
return LiteLLMRealtimeStreamLoggingObject(
usage=usage,
usage=usage.model_copy(update={"seconds": translated_input_seconds})
if translated_input_seconds > 0
else usage,
results=results,
)
@ -3495,6 +3559,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed"
class _LiveResponsePayload(BaseModel):
@ -3717,6 +3782,7 @@ def handle_realtime_stream_cost_calculation(
litellm_logging_obj: LitellmLoggingObject | None = None,
custom_pricing_model: str | None = None,
base_pricing_model: str | None = None,
deployment_models: Sequence[str | None] = (),
) -> float:
"""
Handles the cost calculation for realtime stream responses.
@ -3761,6 +3827,13 @@ def handle_realtime_stream_cost_calculation(
if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
else 0.0
)
translation_cost: Final = handle_realtime_translation_cost_calculation(
results=results,
custom_llm_provider=custom_llm_provider,
litellm_model_name=litellm_model_name,
deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models),
session_model=received_model,
)
live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
live_session_started: Final = any(result["type"] == "session.started" for result in results)
live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
@ -3817,12 +3890,19 @@ def handle_realtime_stream_cost_calculation(
input_cost: Final = input_cost_per_token + live_response_input_cost
output_cost: Final = output_cost_per_token + live_response_output_cost
total_cost: Final = (
input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost
input_cost
+ output_cost
+ transcription_cost
+ translation_cost
+ live_voice_cost
+ audio_duration_cost
+ text_message_cost
)
additional_costs: Final = tuple(
(cost_name, cost)
for cost_name, cost in (
("transcription_cost", transcription_cost),
("translation_cost", translation_cost),
("live_voice_cost", live_voice_cost),
("audio_duration_cost", audio_duration_cost),
("text_message_cost", text_message_cost),
@ -3843,6 +3923,55 @@ def handle_realtime_stream_cost_calculation(
return total_cost
def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]:
usage_events: Final = tuple(
usage
for result in results
if (event := as_str_mapping(result)) is not None
and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
and (usage := as_str_mapping(event.get("usage"))) is not None
)
input_seconds: Final = sum(
seconds
for usage in usage_events
if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None
)
output_seconds: Final = sum(
output_duration
for usage in usage_events
if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None
)
return float(input_seconds), float(output_seconds)
def _translation_per_second_rates(
models: Sequence[str | None], custom_llm_provider: str
) -> tuple[float | None, float | None]:
model_infos: Final = tuple(
_lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None
)
return (
_first_declared_transcription_rate(("input_cost_per_second",), model_infos),
_first_declared_transcription_rate(("output_cost_per_second",), model_infos),
)
def handle_realtime_translation_cost_calculation(
results: OpenAIRealtimeStreamList,
custom_llm_provider: str,
litellm_model_name: str,
deployment_pricing_models: Sequence[str | None] = (),
session_model: str | None = None,
) -> float:
input_seconds, output_seconds = _translation_session_seconds(results)
if input_seconds <= 0 and output_seconds <= 0:
return 0.0
input_rate, output_rate = _translation_per_second_rates(
(*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider
)
return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0)
def handle_realtime_transcription_cost_calculation(
results: OpenAIRealtimeStreamList,
custom_llm_provider: str,
@ -3919,17 +4048,16 @@ def _get_transcription_model_name_from_results(
)
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None:
match key:
case "input_cost_per_second":
return info.get("input_cost_per_second")
case "output_cost_per_second":
return info.get("output_cost_per_second")
case "input_cost_per_audio_token":
return info.get("input_cost_per_audio_token")
case "output_cost_per_audio_token":
return info.get("output_cost_per_audio_token")
case "input_cost_per_token":
return info.get("input_cost_per_token")
case "output_cost_per_token":
@ -3938,7 +4066,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
assert_never(key)
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
@ -3956,15 +4084,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript
)
def _transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
) -> float:
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
return next((rate for rate in rates if rate is not None), 0.0)
def _first_declared_transcription_rate(
keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None]
) -> float | None:
rates: Final = (_declared_transcription_rate(info, keys) for info in infos)
return next((rate for rate in rates if rate is not None), None)
def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float:
return _first_declared_transcription_rate(keys, infos) or 0.0
def _transcription_usage_cost(
usage: dict,
usage: Mapping[str, object],
model_info: ModelInfo | None,
override_info: ModelInfo | None = None,
) -> float:
@ -3973,13 +4105,13 @@ def _transcription_usage_cost(
usage_type: Final = usage.get("type")
if usage_type == "duration":
seconds: Final = usage.get("seconds") or 0.0
seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0
return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info)
if usage_type == "tokens":
input_token_details: Final = usage.get("input_token_details") or {}
audio_tokens: Final = input_token_details.get("audio_tokens") or 0
text_tokens: Final = input_token_details.get("text_tokens") or 0
output_tokens: Final = usage.get("output_tokens") or 0
input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {}
audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0
text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0
output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0
audio_cost: Final = float(audio_tokens) * _transcription_rate(
("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info
)

11
litellm/images/main.py
View file

@ -28,7 +28,7 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params
from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params, header_auth_params
#################### Initialize provider clients ####################
llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler()
@ -988,10 +988,15 @@ def image_edit(
or custom_llm_provider == "azure"
or custom_llm_provider in litellm.openai_compatible_providers
):
dropped_form_params: Final = header_auth_params(custom_llm_provider)
image_edit_request_params.update(
flatten_form_field_values(
non_default_params,
extra_body if isinstance(extra_body, dict) else None,
{key: value for key, value in non_default_params.items() if key not in dropped_form_params},
(
{key: value for key, value in extra_body.items() if key not in dropped_form_params}
if isinstance(extra_body, dict)
else None
),
)
)

433
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -9,24 +9,29 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
"""
import bisect
import copy
import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.prompt_templates.common_utils import (
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_as_cache_control,
with_prompt_cache_breakpoint,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
tool_call_ids_answered_by_client,
@ -85,8 +90,40 @@ _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object])
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues
CostMap: TypeAlias = Mapping[str, Mapping[str, object]]
def routed_model_info(value: object) -> Mapping[str, object] | None:
try:
return _MODEL_INFO_ADAPTER.validate_python(value)
except ValidationError:
return None
def _live_cost_map_unless(model_cost: CostMap | None) -> CostMap:
import litellm
if model_cost is not None:
return model_cost
return cast(CostMap, litellm.model_cost) # cast-ok: the live map holds catalog entries
def declared_prompt_cache_breakpoint(model_info: Mapping[str, object] | None) -> bool | None:
declared: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
return declared if isinstance(declared, bool) else None
def routed_cost_map() -> CostMap:
from litellm.utils import catalog_with_runtime_registrations
return catalog_with_runtime_registrations()
def _catalog_unless_unrouted(model_info: Mapping[str, object] | None) -> CostMap | None:
return None if model_info is None else routed_cost_map()
def _validated_object_mapping(value: object) -> dict[object, object] | None:
@ -295,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object:
return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params)
def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]:
if not isinstance(value, (list, tuple)):
return ()
@ -314,19 +357,27 @@ def _served_provider_for_model(model: str, api_base: object = None) -> str | Non
return provider
def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool:
return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True
def supports_openai_prompt_cache_breakpoint(
model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool:
return openai_prompt_cache_breakpoint_support(model, custom_llm_provider, model_info) is True
def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None:
def openai_prompt_cache_breakpoint_support(
model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool | None:
declared: Final = declared_prompt_cache_breakpoint(model_info)
if declared is not None:
return declared
model_cost: Final = _catalog_unless_unrouted(model_info)
hosted_flag: Final = (
None
if custom_llm_provider is None
else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model)
else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model, model_cost)
)
if hosted_flag is not None:
return hosted_flag
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model)
model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model, model_cost)
if model_map_flag is not None:
return model_map_flag
fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1])
@ -347,13 +398,18 @@ class _DeclaredBaseModels(BaseModel):
return tuple(name for name in candidates if isinstance(name, str) and name)
def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None:
return routed_model_info(_DeclaredBaseModels.model_validate(litellm_params).model_info)
def deployment_prompt_cache_breakpoint_support(
model: str, custom_llm_provider: str, litellm_params: object
) -> bool | None:
"""``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
model_info: Final = routed_model_info(declared.model_info)
supports: Final = (
openai_prompt_cache_breakpoint_support(name, custom_llm_provider)
openai_prompt_cache_breakpoint_support(name, custom_llm_provider, model_info)
for name in dict.fromkeys((*declared.names(), model))
)
return next((support for support in supports if support is not None), None)
@ -370,11 +426,10 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None:
import litellm
entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1]))
flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict))
def _model_map_prompt_cache_breakpoint_flag(model: str, model_cost: CostMap | None = None) -> bool | None:
cost_map: Final = _live_cost_map_unless(model_cost)
entries: Final = (cost_map.get(key) for key in (model, model.rsplit("/", 1)[-1]))
flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, Mapping))
return next((flag is True for flag in flags if flag is not None), None)
@ -390,7 +445,7 @@ def _hosted_openai_dialect_flag(
model: str,
custom_llm_provider: str | None,
resolve_provider: Callable[[str], str | None],
model_cost: Mapping[str, Mapping[str, object]] | None = None,
model_cost: CostMap | None = None,
) -> bool | None:
"""
Explicit opt-in for an OpenAI-shaped deployment served by another provider.
@ -407,13 +462,7 @@ def _hosted_openai_dialect_flag(
row of the same name, so an exact entry only speaks for a deployment when it is
keyed for that deployment's provider.
"""
import litellm
cost_map: Final = (
cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries
if model_cost is None
else model_cost
)
cost_map: Final = _live_cost_map_unless(model_cost)
exact_entry: Final = cost_map.get(model)
exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None
if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)):
@ -423,10 +472,16 @@ def _hosted_openai_dialect_flag(
return None
if isinstance(exact_entry, Mapping) and exact_provider == provider:
return _hosted_entry_flag(exact_entry, lambda: provider)
entries: Final = (cost_map.get(candidate) for candidate in _provider_cost_map_keys(model, provider))
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
def _provider_cost_map_keys(model: str, provider: str) -> tuple[str, ...]:
from litellm.utils import get_potential_model_names
names: Final = get_potential_model_names(model, provider)
candidates: Final = (
return (
names["combined_model_name"],
names["region_free_combined_model_name"],
names["split_model"],
@ -434,9 +489,60 @@ def _hosted_openai_dialect_flag(
names["stripped_model_name"],
names["provider_prefixed_model_name"],
)
entries: Final = (cost_map.get(candidate) for candidate in candidates)
flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping))
return next((flag for flag in flags if flag is not None), None)
def _cost_map_lists_model_for_provider(model: str, provider: str, model_cost: CostMap) -> bool:
entries: Final = (model_cost.get(key) for key in (model, *_provider_cost_map_keys(model, provider)))
return any(isinstance(entry, Mapping) and entry.get("litellm_provider") == provider for entry in entries)
_PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT: Final = frozenset(("bedrock", "bedrock_converse"))
_PROVIDERS_HOSTING_OPENAIS_API: Final = frozenset(("azure", "azure_ai", "bedrock_mantle"))
def _declared_hosted_flag(
declared: bool, model: str, provider: str | None, model_cost: CostMap, deployment_provider: object
) -> bool | None:
"""
A deployment's own ``supports_prompt_cache_breakpoint`` picks OpenAI's breakpoint only where the
request goes out OpenAI-shaped. Bedrock reads the same flag on every route as whether Converse takes
a ``cachePoint``, and a provider building Anthropic messages takes ``cache_control``, so both keep
the Anthropic mark. A gateway or self-hosted server (OpenRouter, a LiteLLM proxy, vLLM) may pass
the request on to Anthropic, so there the flag only picks OpenAI's breakpoint for a model the cost
map, or the deployment's own ``litellm_provider``, lists under that provider.
"""
if provider is None or provider == "openai":
return None
return (
declared
and provider not in _PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT
and (
provider in _PROVIDERS_HOSTING_OPENAIS_API
or deployment_provider == provider
or _cost_map_lists_model_for_provider(model, provider, model_cost)
)
and _sends_openai_shaped_chat(model, provider)
)
@lru_cache(maxsize=512)
def _sends_openai_shaped_chat(model: str, custom_llm_provider: str) -> bool:
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata
from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.openai.openai import OpenAIConfig
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
try:
served_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider)
chat_config: Final = ProviderConfigManager.get_provider_chat_config(
model=served_model, provider=LlmProviders(provider)
)
except (BadRequestError, ValueError):
return False
return isinstance(chat_config, (OpenAIGPTConfig, OpenAIConfig, AzureOpenAIConfig))
def targets_openai_api(api_base: object) -> bool:
@ -456,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool:
return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS)
def _carries_cache_control(block: object) -> bool:
return _attribute_or_key(block, "cache_control") is not None
def _attribute_or_key(value: object, key: str) -> object | None:
if hasattr(value, key):
return _ATTRIBUTE_VALUE.validate_python(getattr(value, key))
@ -535,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b
return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig)
def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool:
return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages(
model, custom_llm_provider
)
def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None:
provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP)
return provider if isinstance(provider, str) else None
@ -578,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
def _carries_prompt_cache_breakpoint(block: object) -> bool:
return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None
def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") != "assistant"
def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") == "tool" and any(
_carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()
)
def _part_without_prompt_cache_breakpoint(part: object) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return {
key: value
for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items()
if key != "prompt_cache_breakpoint"
}
def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
if not _tool_result_carries_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
return cast( # cast-ok: the same tool message with its part marks removed
AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]}
)
def _created_cache_control(original: object, translated: object) -> bool:
return (
_attribute_or_key(original, "cache_control") is None
and _attribute_or_key(translated, "cache_control") is not None
)
def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]:
if translated is original:
return ()
original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or []
translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or []
return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts))
def _part_without_cache_control(part: object) -> object:
return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"}
def _message_dropping_created_cache_controls_between(
message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int
) -> AllMessageValues:
if not any(created):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before)
return cast( # cast-ok: the same message with its middle translated marks removed
AllMessageValues,
{
**message,
"content": [
_part_without_cache_control(part)
if is_created and first_dropped <= ordinal < first_kept_after
else part
for part, is_created, ordinal in zip(parts, created, ordinals)
],
},
)
# Set by a caller whose message list is not the one that goes upstream -- today the
# Responses API layer, whose `instructions` only becomes a system message further down.
# Tells this hook to hand role-targeted points to the pass holding the final messages
@ -667,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if not injection_points:
return model, messages, non_default_params
# Create a deep copy of messages to avoid modifying the original list
processed_messages = copy.deepcopy(messages)
message_points: Final = tuple(
cast(CacheControlMessageInjectionPoint, point)
for point in injection_points
@ -677,18 +859,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
remaining_points: Final = tuple(point for point in injection_points if point.get("location") != "message")
custom_llm_provider: Final = _custom_llm_provider(non_default_params, injection_points[0])
request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict
custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0])
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
model_info: Final = routed_model_info(request_params.get("model_info"))
openai_dialect: Final = (
stamped_dialect
if isinstance(stamped_dialect, bool)
else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model,
custom_llm_provider,
non_default_params.get("api_base") or non_default_params.get("base_url"),
non_default_params.get("prompt_cache_options"),
request_params.get("api_base") or request_params.get("base_url"),
request_params.get("prompt_cache_options"),
model_info,
)
)
processed_messages = copy.deepcopy(
[without_prompt_cache_breakpoints(message) for message in messages]
if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider)
else messages
)
# A provisional message list defers every role-targeted point to the pass holding
# the final one: a role with no message here may have one there, and settling all
# of them in one pass is what lets config order decide the shared breakpoint
@ -757,32 +947,35 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider: str | None,
api_base: object = None,
prompt_cache_options: object = None,
model_info: Mapping[str, object] | None = None,
) -> bool:
if model is None:
return False
hosted_flag: Final = _hosted_openai_dialect_flag(
model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base)
declared: Final = declared_prompt_cache_breakpoint(model_info)
hosted_flag: Final = (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base),
_catalog_unless_unrouted(model_info),
)
if declared is None
else _declared_hosted_flag(
declared,
model,
custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base),
routed_cost_map(),
None if model_info is None else model_info.get("litellm_provider"),
)
)
if hosted_flag is not None:
return hosted_flag
if not supports_openai_prompt_cache_breakpoint(model):
return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info):
return False
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
return False
return prompt_cache_options is not None or targets_openai_api(api_base)
@staticmethod
def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool:
return (
_hosted_openai_dialect_flag(
model,
custom_llm_provider,
AnthropicCacheControlHook._resolve_provider,
GetModelCostMap.loaded_model_cost_map(),
)
is True
)
@staticmethod
def _resolve_provider(model: str, api_base: object = None) -> str | None:
return _served_provider_for_model(model, api_base)
@ -829,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return int(wire_cache_control is not None) + tool_blocks + envelope_blocks
@staticmethod
def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]:
return tuple(
tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool)
)
@staticmethod
def count_chat_external_cache_breakpoints(optional_params: object) -> int:
params: Final = _validated_object_mapping(optional_params) or {}
return AnthropicCacheControlHook.count_external_cache_breakpoints(
_as_object_list(params.get("tools")), params.get("cache_control")
)
@staticmethod
def count_external_cache_breakpoints_on_messages_route(
tools: Iterable[object] | None, cache_control: object, request_kwargs: object
@ -984,6 +1190,16 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return message_count + content_count + tool_call_count
@staticmethod
def count_converse_cache_points(message: AllMessageValues) -> int:
content: Final = _as_object_list(_attribute_or_key(message, "content"))
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or []
marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block))
marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call))
if message.get("role") == "tool":
return int(_carries_cache_control(message) or marked_parts > 0)
return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls
@staticmethod
def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool:
"""Return True if the message already carries any cache_control."""
@ -1042,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return renamed
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
@staticmethod
def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
"""Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's
``cache_control``. Marks with no OpenAI slot are left for the caller to strip."""
if not _may_carry_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content"))
if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts):
return message
return cast( # cast-ok: the same message with its part marks renamed
AllMessageValues,
{**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]},
)
@staticmethod
def prompt_cache_breakpoints_dropped_from_tool_results(
messages: Sequence[AllMessageValues], drop_params: bool, model: str
) -> tuple[AllMessageValues, ...]:
if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages):
return tuple(messages)
if not drop_params:
raise UnsupportedParamsError(
message=(
f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool "
f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model)
return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages)
@staticmethod
def prompt_cache_breakpoints_as_capped_cache_control(
messages: Sequence[AllMessageValues],
drop_params: bool,
sent_breakpoints: Callable[[AllMessageValues], int],
external_breakpoints: Callable[[], int],
) -> tuple[AllMessageValues, ...]:
"""``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's
breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends.
Under ``drop_params`` the first and last translated marks fill the slots the client's own
``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest
turns stay cached, and without it a request over the cap is rejected."""
translated: Final = tuple(
AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params)
for message in messages
)
if all(after is before for before, after in zip(messages, translated)):
return translated
created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated))
created_marks: Final = sum(sum(flags) for flags in created)
if created_marks == 0:
return translated
external: Final = external_breakpoints()
breakpoints: Final = sum(map(sent_breakpoints, translated)) + external
excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS
if excess <= 0:
return translated
if not drop_params:
raise UnsupportedParamsError(
message=(
f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this "
f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} "
f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them."
),
status_code=400,
)
verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params")
marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0))
def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]:
first_dropped: Final = (kept_marks + 1) // 2
first_kept_after: Final = created_marks - kept_marks // 2
return tuple(
_message_dropping_created_cache_controls_between(
message, flags, before, first_dropped, first_kept_after
)
for message, flags, before in zip(translated, created, marks_before)
)
def over_cap(kept_marks: int) -> bool:
return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS
first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap)
return capped_request(max(0, first_over_cap - 1))
@staticmethod
def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
if not _may_carry_prompt_cache_breakpoint(message):
@ -1181,6 +1483,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider: str | None,
api_base: object,
prompt_cache_options: object,
model_info: Mapping[str, object] | None = None,
) -> Sequence[Mapping[str, object]]:
"""Carry onto the points what the prompt-management hook never receives.
@ -1192,7 +1495,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
across requests.
"""
with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect(
points, model, custom_llm_provider, api_base, prompt_cache_options
points, model, custom_llm_provider, api_base, prompt_cache_options, model_info
)
stamped_earlier: Final = any(EXTERNAL_BREAKPOINTS_STAMP in point for point in with_dialect)
if external_breakpoints == 0 and not stamped_earlier:
@ -1206,14 +1509,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider: str | None,
api_base: object,
prompt_cache_options: object,
model_info: Mapping[str, object] | None = None,
) -> Sequence[Mapping[str, object]]:
if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider):
if model_info is None and not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(
model, custom_llm_provider
):
return points
return AnthropicCacheControlHook._stamped(
points,
"_litellm_openai_dialect",
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, prompt_cache_options
model, custom_llm_provider, api_base, prompt_cache_options, model_info
),
)
@ -1272,6 +1578,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
cache_control: object = None,
request_kwargs: object = None,
on_messages_route: bool = False,
model_info: Mapping[str, object] | None = None,
) -> list[CacheControlInjectionPoint]:
"""Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
@ -1289,7 +1596,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True:
return []
if not supports_anthropic_cache_control(model, custom_llm_provider):
if not supports_anthropic_cache_control(model, custom_llm_provider, model_info):
return []
if AnthropicCacheControlHook._request_has_cache_control(
@ -1367,6 +1674,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
tools: list | None = None,
enable_prompt_caching: bool | None = None,
api_base: object = None,
model_info: Mapping[str, object] | None = None,
) -> None:
"""For /chat/completions: resolve the injection points the request should carry.
@ -1381,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points"))
if configured:
tools_keeping_marks: Final = tuple(
tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool)
)
tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools)
stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook(
configured,
AnthropicCacheControlHook.count_external_cache_breakpoints(
@ -1393,6 +1699,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider,
api_base,
non_default_params.get("prompt_cache_options"),
model_info,
)
non_default_params["cache_control_injection_points"] = (
AnthropicCacheControlHook._stamped(stamped, CUSTOM_LLM_PROVIDER_STAMP, custom_llm_provider)
@ -1409,6 +1716,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
enable_prompt_caching=enable_prompt_caching,
cache_control=non_default_params.get("cache_control"),
request_kwargs=non_default_params,
model_info=model_info,
)
if points:
non_default_params["cache_control_injection_points"] = points
@ -1527,6 +1835,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
cache_control=cache_control,
request_kwargs=kwargs,
on_messages_route=True,
model_info=routed_model_info(kwargs.get("model_info")),
)
if model is not None
else ()
@ -1535,7 +1844,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return messages, system
openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
model,
custom_llm_provider,
api_base,
kwargs.get("prompt_cache_options"),
routed_model_info(kwargs.get("model_info")),
)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
@ -1669,9 +1982,17 @@ def messages_within_cache_control_cap(
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
return messages_within_reserved_cache_control_cap(
messages,
AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
),
)
def messages_within_reserved_cache_control_cap(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:

13
litellm/integrations/custom_guardrail.py
View file

@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger):
enrich_http_exception_with_guardrail_context(result, self)
raise result
def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool:
return (
isinstance(request_data.get("guardrails"), list)
and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True
)
async def async_post_call_success_deployment_hook(
self,
request_data: dict,
@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger):
"""
Allow modifying / reviewing the response just after it's received from the deployment.
"""
# should run guardrail
litellm_guardrails: Final = request_data.get("guardrails")
if litellm_guardrails is None or not isinstance(litellm_guardrails, list):
return None
if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True:
if not self.runs_post_call_deployment_hook(request_data):
return None
target: Final = self._deployment_hook_target()

341
litellm/litellm_core_utils/audio_utils/transcription_streaming.py Normal file
View file

@ -0,0 +1,341 @@
from __future__ import annotations
import datetime
import json
import traceback
from collections.abc import AsyncIterator, Callable, Iterator, Mapping
from typing import Final, Protocol
import anyio
from openai import AsyncStream, Stream
from openai.types.audio import (
TranscriptionStreamEvent,
TranscriptionTextDeltaEvent,
TranscriptionTextDoneEvent,
)
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
)
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
_STREAM_FLAG: Final = TypeAdapter(bool)
def transcription_stream_flag(value: object) -> bool | None:
try:
return _STREAM_FLAG.validate_python(value)
except ValidationError:
return None
def transcription_stream_unset(value: object) -> bool:
return value is None or value == ""
def transcription_stream_requested(value: object) -> bool:
return transcription_stream_flag(value) is True
TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n"
TranscriptionStreamExceptionMapper = Callable[[Exception], Exception]
def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper:
from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base
model: Final = call_details.get("model")
custom_llm_provider: Final = call_details.get("custom_llm_provider")
extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params"))
def map_exception(exception: Exception) -> Exception:
try:
mapped: Final[object] = exception_type(
model=model if isinstance(model, str) else None,
original_exception=exception,
custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
extra_kwargs=extra_kwargs,
)
except Exception as mapped_exception: # noqa: BLE001 # exception_type raises the mapped exception
return mapped_exception
return mapped if isinstance(mapped, Exception) else exception
return map_exception
def _guarded_transcription_event_payloads(
response: TranscriptionResponse,
) -> tuple[dict[str, object], dict[str, object]]:
text: Final = response.text or ""
usage: Final = response.usage
languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None))
return (
{"type": "transcript.text.delta", "delta": text},
{
"type": "transcript.text.done",
"text": text,
**({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}),
**({} if languages is None else {"languages": [language.model_dump() for language in languages]}),
},
)
def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]:
return (
*(
f"data: {json.dumps(event, separators=(',', ':'))}\n\n"
for event in _guarded_transcription_event_payloads(response)
),
TRANSCRIPTION_STREAM_DONE_FRAME,
)
class GuardedTranscriptionReplay:
def __init__(self, response: TranscriptionResponse) -> None:
self.response: Final = response
self._hidden_params: dict[str, object] = response.hidden_params
def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]:
return self._events()
async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]:
delta, done = _guarded_transcription_event_payloads(self.response)
yield TranscriptionTextDeltaEvent.model_construct(**delta)
yield TranscriptionTextDoneEvent.model_construct(**done)
class TranscriptionStreamLogging(Protocol):
def success_handler(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
async def async_success_handler(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def handle_sync_success_callbacks_for_async_calls(
self,
result: TranscriptionResponse,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
*,
for_async_request: bool = False,
) -> None: ...
async def async_failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None: ...
def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
class _TranscriptionEventCollector:
def __init__(self, duration: float | None) -> None:
self.duration = duration
self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event
self.done_event: TranscriptionTextDoneEvent | None = None
def add(self, event: TranscriptionStreamEvent) -> None:
if isinstance(event, TranscriptionTextDeltaEvent):
self.text_deltas.append(event.delta)
elif isinstance(event, TranscriptionTextDoneEvent):
self.done_event = event
def response(self) -> TranscriptionResponse:
done_event: Final = self.done_event
response: Final = TranscriptionResponse(
text=done_event.text if done_event is not None else "".join(self.text_deltas),
usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None,
languages=getattr(done_event, "languages", None) if done_event is not None else None,
)
if self.duration is not None:
response.set_audio_transcription_duration(self.duration)
return response
def _billed_usage(response: TranscriptionResponse) -> Usage:
usage: Final[object] = getattr(response, "usage", None)
token_usage: Final = (
TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
if isinstance(usage, TranscriptionUsageTokensObject)
else None
)
return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
def _record_billable_partial_spend(
logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
) -> None:
partial_response: Final = collector.response()
partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
if partial_cost <= 0:
return
logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
def __init__(
self,
stream: Stream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> None:
self.__dict__.update(stream.__dict__)
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
"additional_headers": process_response_headers(stream.response.headers)
}
self._logging_obj = logging_obj
self._start_time = start_time
self._map_exception = map_exception
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
self._finalized = False
self._failed = False
source_iterator: Final = self._iterator
self._iterator = self._logging_iterator(source_iterator)
def _logging_iterator(
self, source_iterator: Iterator[TranscriptionStreamEvent]
) -> Iterator[TranscriptionStreamEvent]:
try:
for event in source_iterator:
self._collector.add(event)
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
raise mapped_exception from exception
finally:
self._finalize()
def _finalize(self) -> None:
if self._finalized or self._failed:
return
self._finalized = True
self._logging_obj.success_handler(
self._collector.response(),
self._start_time,
datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract
)
def close(self) -> None:
try:
super().close()
finally:
self._finalize()
@property
def logging_obj(self) -> TranscriptionStreamLogging:
return self._logging_obj
class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
def __init__(
self,
stream: AsyncStream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> None:
self.__dict__.update(stream.__dict__)
self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata
"additional_headers": process_response_headers(stream.response.headers)
}
self._logging_obj = logging_obj
self._start_time = start_time
self._map_exception = map_exception
self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None))
self._finalized = False
self._failed = False
source_iterator: Final = self._iterator
self._iterator = self._logging_iterator(source_iterator)
async def _logging_iterator(
self, source_iterator: AsyncIterator[TranscriptionStreamEvent]
) -> AsyncIterator[TranscriptionStreamEvent]:
try:
async for event in source_iterator:
self._collector.add(event)
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(
mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True
)
await self._logging_obj.async_failure_handler(
mapped_exception,
traceback.format_exc(),
self._start_time,
end_time,
)
raise mapped_exception from exception
finally:
await self._finalize()
async def _finalize(self) -> None:
if self._finalized or self._failed:
return
self._finalized = True
response: Final = self._collector.response()
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.handle_sync_success_callbacks_for_async_calls(
result=response,
start_time=self._start_time,
end_time=end_time,
)
with anyio.CancelScope(shield=True):
await self._logging_obj.async_success_handler(
result=response,
start_time=self._start_time,
end_time=end_time,
)
async def close(self) -> None:
try:
await super().close()
finally:
await self._finalize()
@property
def logging_obj(self) -> TranscriptionStreamLogging:
return self._logging_obj
def wrap_transcription_stream(
stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent],
logging_obj: TranscriptionStreamLogging,
start_time: datetime.datetime,
map_exception: TranscriptionStreamExceptionMapper,
) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream:
if isinstance(stream, AsyncStream):
return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception)
return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception)

10
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -21,6 +21,16 @@ from litellm.types.files import (
from litellm.types.utils import FileTypes
def normalized_audio_duration_seconds(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float)):
return None
try:
seconds: Final = float(value)
except OverflowError:
return None
return seconds if math.isfinite(seconds) and seconds >= 0 else None
@dataclass
class ProcessedAudioFile:
"""

10
litellm/litellm_core_utils/core_helpers.py
View file

@ -97,7 +97,7 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
return float(seconds / denominator)
_DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
_LAX_BOOL: Final = TypeAdapter(bool)
def as_str_mapping(value: object) -> Mapping[str, object] | None:
@ -160,15 +160,19 @@ def attribute_items(source: object, name: str) -> Iterable[object]:
return iter_items(value) if value else ()
def normalize_drop_params(value: object) -> bool | None:
def lax_bool(value: object) -> bool | None:
if value is None or isinstance(value, bool):
return value
try:
return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
return _LAX_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
except ValidationError:
return None
def normalize_drop_params(value: object) -> bool | None:
return lax_bool(value)
def drops_unsupported_params(drop_params: object) -> bool:
return litellm.drop_params is True or normalize_drop_params(drop_params) is True

116
litellm/litellm_core_utils/litellm_logging.py
View file

@ -19,9 +19,11 @@ from datetime import datetime as dt_object
from enum import Enum
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast
from httpx import Response, Timeout
from openai import AsyncStream as OpenAIAsyncStream
from openai import Stream as OpenAIStream
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
@ -97,6 +99,7 @@ from litellm.litellm_core_utils.hidden_params import (
get_hidden_params,
served_from_cache,
set_hidden_param,
set_hidden_params,
)
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
@ -122,6 +125,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import (
diagnose_zero_cost,
zero_cost_warning,
)
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler
from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages,
truncate_base64_in_messages_async,
@ -142,7 +146,7 @@ from litellm.litellm_core_utils.served_output_texts import (
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
@ -300,6 +304,11 @@ class _EnterpriseAlertingLoggers(NamedTuple):
smtp_email: type[CustomLogger]
class BridgedChatRequest(NamedTuple):
optional_params: Mapping[str, object]
litellm_params: Mapping[str, object]
@functools.cache
def _enterprise_alerting_loggers() -> _EnterpriseAlertingLoggers:
try:
@ -580,6 +589,7 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
_REDACTED_RESULT: Final = TypeAdapter(object)
_LoggedResponse: Final = TypeVar("_LoggedResponse")
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -873,6 +883,7 @@ class Logging(LiteLLMLoggingBaseClass):
# once that response is priced, the same way a non-streamed response is logged
self.client_facing_stream_model: str | None = None
self.client_facing_response_id: str | None = None
self.bridged_chat_request: BridgedChatRequest | None = None
self.zero_cost_warned: bool = False
self.llm_caching_handler: LLMCachingHandler | None = None
@ -2130,6 +2141,9 @@ class Logging(LiteLLMLoggingBaseClass):
if resolved_cache_hit is True:
return 0.0
if isinstance(result, (OpenAIStream, OpenAIAsyncStream)):
return None
if is_unbilled_non_inference_call(
self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result
):
@ -3230,6 +3244,40 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"model": model})
def _as_served_chat_response(self, response: _LoggedResponse) -> _LoggedResponse | ModelResponse:
request: Final = self.bridged_chat_request
if request is None or not isinstance(response, ResponsesAPIResponse):
return response
return self._bridged_chat_response(response, request)
def _as_served_response(
self, response: _LoggedResponse
) -> _LoggedResponse | ModelResponse | TextCompletionResponse:
request: Final = self.bridged_chat_request
if request is None or not isinstance(response, ResponsesAPIResponse):
return response
served: Final = self._bridged_chat_response(response, request)
if self.call_type not in (CallTypes.text_completion.value, CallTypes.atext_completion.value): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
return served
return self._as_text_completion_priced_as_chat(served)
def _as_text_completion_priced_as_chat(self, served: ModelResponse) -> TextCompletionResponse:
text_completion: Final = LiteLLMResponseObjectHandler.convert_chat_to_text_completion(
served, TextCompletionResponse()
)
set_hidden_params(text_completion, dict(get_hidden_params(served) or {}))
return text_completion
def _bridged_chat_response(self, response: ResponsesAPIResponse, request: BridgedChatRequest) -> ModelResponse:
translated: Final = self._translate_responses_api_response_to_model_response(response, request)
served: Final = translated.model_copy(
update={
"id": ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response.id)
}
)
set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})})
return served
def _with_continued_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
@ -3311,6 +3359,8 @@ class Logging(LiteLLMLoggingBaseClass):
)
result = self.normalize_endpoint_logging_result(result)
if self.stream is not True:
result = self._as_served_response(result) # rebind-ok: bridged chat calls log the served response
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@ -3625,6 +3675,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
start_time=start_time,
end_time=end_time,
@ -3639,7 +3690,12 @@ class Logging(LiteLLMLoggingBaseClass):
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
ModelResponse
| TextCompletionResponse
| ResponsesAPIResponse
| InteractionsAPIResponse
| TranscriptionResponse
| None
) = None
if "complete_streaming_response" in self.model_call_details and not for_async_request:
return # break out of this.
@ -3655,17 +3711,21 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["complete_streaming_response"] = complete_streaming_response
self._surface_response_headers_from_result(complete_streaming_response)
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
result=complete_streaming_response if provider_transcription is None else provider_transcription
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(
self._as_served_chat_response(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
logged_streaming_response if provider_transcription is None else provider_transcription,
start_time,
end_time,
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
@ -4192,6 +4252,7 @@ class Logging(LiteLLMLoggingBaseClass):
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
)
)
provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None
start_time, end_time, result = self._success_handler_helper_fn(
start_time=start_time,
end_time=end_time,
@ -4211,9 +4272,7 @@ class Logging(LiteLLMLoggingBaseClass):
## BUILD COMPLETE STREAMED RESPONSE
if "async_complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response: Final[
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
] = self._get_assembled_streaming_response(
complete_streaming_response: Final = self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
@ -4235,7 +4294,7 @@ class Logging(LiteLLMLoggingBaseClass):
get_base_model_from_metadata(model_call_details=self.model_call_details)
# base_model defaults to None if not set on model_info
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
result=complete_streaming_response if provider_transcription is None else provider_transcription
)
verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"])
@ -4256,13 +4315,17 @@ class Logging(LiteLLMLoggingBaseClass):
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(
self._as_served_chat_response(complete_streaming_response)
)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
try:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
logged_streaming_response if provider_transcription is None else provider_transcription,
start_time,
end_time,
)
except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release)
verbose_logger.exception(
@ -5132,10 +5195,17 @@ class Logging(LiteLLMLoggingBaseClass):
end_time: datetime.datetime,
is_async: bool,
streaming_chunks: list[object],
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
) -> (
ModelResponse
| TextCompletionResponse
| ResponsesAPIResponse
| InteractionsAPIResponse
| TranscriptionResponse
| None
):
if self.stream is not True:
return None
if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)):
return result
elif isinstance(
result,
@ -5206,6 +5276,13 @@ class Logging(LiteLLMLoggingBaseClass):
row is built from this response rather than re-parsed from the served body."""
self.bridged_model_response = response
def record_bridged_chat_request(
self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> None:
self.bridged_chat_request = BridgedChatRequest(
MappingProxyType(dict(optional_params)), MappingProxyType(dict(litellm_params))
)
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
@ -5277,6 +5354,9 @@ class Logging(LiteLLMLoggingBaseClass):
):
result = result.response
if isinstance(result, ResponsesAPIResponse):
bridged_request: Final = self.bridged_chat_request
if bridged_request is not None:
return self._bridged_chat_response(result, bridged_request)
return self._translate_responses_api_response_to_model_response(result)
raw_body: Final[object] = result
@ -5328,7 +5408,9 @@ class Logging(LiteLLMLoggingBaseClass):
set_response_cost_in_hidden_params(result, reported_cost)
return result
def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse:
def _translate_responses_api_response_to_model_response(
self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None
) -> ModelResponse:
"""
Convert a Responses API response into a ModelResponse for spend_logs.
@ -5351,8 +5433,8 @@ class Logging(LiteLLMLoggingBaseClass):
logging_obj=self,
request_data={},
messages=[],
optional_params={},
litellm_params={},
optional_params={} if request is None else dict(request.optional_params),
litellm_params={} if request is None else dict(request.litellm_params),
encoding=litellm.encoding,
)
translated.id = served_id or translated.id

79
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1040,6 +1040,11 @@ def calculate_cache_writing_cost(
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_hit_audio_tokens: ReadOnly[int]
cached_text_tokens: ReadOnly[int]
cached_audio_tokens: ReadOnly[int]
cached_image_tokens: ReadOnly[int]
has_cached_tokens_details: ReadOnly[bool]
cache_creation_tokens: int
cache_creation_token_details: CacheCreationTokenDetails | None
text_tokens: int
@ -1126,6 +1131,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_hit_audio_tokens=cached_audio_tokens,
cached_text_tokens=cached_text_tokens,
cached_audio_tokens=cached_audio_tokens,
cached_image_tokens=cached_image_tokens,
has_cached_tokens_details=cached_tokens_details is not None,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
@ -1234,15 +1243,11 @@ def _calculate_input_cost(
prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"]
audio_cache_read_rate: Final = get_cost_per_unit(
model_info,
get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier),
None,
)
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost
prompt_cost += float(cache_hit_audio_tokens) * (
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
prompt_cost += _calculate_cache_read_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=model_info,
cache_read_cost=cache_read_cost,
service_tier=service_tier,
)
### AUDIO COST
@ -1331,6 +1336,38 @@ def _calculate_input_cost(
return prompt_cost
def _calculate_cache_read_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
cache_read_cost: float,
service_tier: str | None,
) -> float:
cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"]
cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"]
cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"]
classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0)
total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost
float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost
)
if cached_audio_tokens:
cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier)
cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost)
total_cost += ( # rebind-ok: cached audio contributes to cache-read cost
float(cached_audio_tokens) * float(cached_audio_cost or 0.0)
)
if cached_image_tokens:
cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier)
cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost)
total_cost += ( # rebind-ok: cached images contribute to cache-read cost
float(cached_image_tokens) * float(cached_image_cost or 0.0)
)
return total_cost
def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
"""
Resolve the per-model regional-processing uplift multiplier for a given
@ -1502,6 +1539,10 @@ def generic_cost_per_token(
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_hit_audio_tokens=0,
cached_text_tokens=0,
cached_audio_tokens=0,
cached_image_tokens=0,
has_cached_tokens_details=False,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
@ -1680,6 +1721,7 @@ class BilledTokenRates:
cache_creation_input_token_cost: float
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
output_cost_per_reasoning_token: float
cache_read_input_image_token_cost: float | None = None
def scaled(self, multiplier: float) -> "BilledTokenRates":
if multiplier == 1.0:
@ -1694,6 +1736,11 @@ class BilledTokenRates:
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
),
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
cache_read_input_image_token_cost=(
self.cache_read_input_image_token_cost * multiplier
if self.cache_read_input_image_token_cost is not None
else None
),
)
@ -1802,6 +1849,11 @@ def _cost_map_billed_rates(
cache_creation_input_token_cost=base_costs.cache_creation_rate,
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
output_cost_per_reasoning_token=reasoning_rate,
cache_read_input_image_token_cost=_get_cost_per_unit(
model_info,
_get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier),
None,
),
).scaled(multiplier)
@ -1882,6 +1934,12 @@ def get_token_type_cost_breakdown(
cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(
usage
)
cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"]
image_cache_read_rate: Final = (
rates.cache_read_input_image_token_cost
if rates.cache_read_input_image_token_cost is not None
else rates.cache_read_input_token_cost
)
cache_creation_cost: Final = (
float(cache_creation_tokens) * rates.cache_creation_input_token_cost
if custom_cost_per_token is not None
@ -1895,8 +1953,9 @@ def get_token_type_cost_breakdown(
return TokenTypeCostBreakdown(
reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
cache_read_cost=(
float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost
float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost
+ float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost
+ float(cached_image_tokens) * image_cache_read_rate
),
cache_creation_cost=cache_creation_cost,
rates=rates,

12
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -970,9 +970,11 @@ def convert_to_model_response_object(
model_response_object.data = response_object["data"]
if "usage" in response_object and response_object["usage"] is not None:
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
embedding_usage: Final = model_response_object.usage or Usage()
embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0)
model_response_object.usage = embedding_usage
if start_time is not None and end_time is not None:
model_response_object._response_ms = (
@ -1015,6 +1017,10 @@ def convert_to_model_response_object(
if key in response_object:
setattr(model_response_object, key, response_object[key])
detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages"))
if detected_languages is not None:
model_response_object.languages = detected_languages
if "usage" in response_object and response_object["usage"] is not None:
tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None

8
litellm/litellm_core_utils/model_param_helper.py
View file

@ -1,6 +1,6 @@
from collections.abc import Mapping
from functools import lru_cache
from typing import ClassVar, Final
from typing import ClassVar, Final, get_args
from openai.types.chat.completion_create_params import (
CompletionCreateParamsNonStreaming,
@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.types.llms.anthropic import AnthropicMessagesRequest
from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
from litellm.types.rerank import RerankRequest
@ -178,6 +179,7 @@ class ModelParamHelper:
This follows the OpenAI API Spec
"""
litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams))
try:
from openai.types.audio.transcription_create_params import (
TranscriptionCreateParamsNonStreaming,
@ -187,11 +189,11 @@ class ModelParamHelper:
non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys())
streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys())
all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs)
all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs)
return all_transcription_kwargs
except Exception as e:
verbose_logger.debug("Error getting transcription kwargs %s", str(e))
return set()
return litellm_transcription_kwargs
@staticmethod
def _get_litellm_supported_responses_api_kwargs() -> set[str]:

16
litellm/litellm_core_utils/pattern_match_deployments.py
View file

@ -5,6 +5,7 @@ Class to handle llm wildcard routing and regex pattern matching
import copy
import itertools
import re
from collections.abc import Mapping
from re import Match
from typing import Final
@ -22,6 +23,16 @@ class _RoutedDeployment(BaseModel):
_ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...])
_PATTERN_WIDE_CACHE_FLAGS: Final = frozenset(("supports_prompt_cache_breakpoint", "supports_prompt_caching"))
_AZURE_AI_PATTERN_CACHE_FLAGS: Final = frozenset(("supports_prompt_caching",))
def _cache_flags_left_to_the_catalog(litellm_params: Mapping[str, object]) -> frozenset[str]:
model: Final = litellm_params.get("model")
provider: Final = litellm_params.get("custom_llm_provider") or inferred_provider(
model if isinstance(model, str) else None
)
return _AZURE_AI_PATTERN_CACHE_FLAGS if provider == "azure_ai" else _PATTERN_WIDE_CACHE_FLAGS
class PatternUtils:
@ -133,6 +144,11 @@ class PatternMatchRouter:
matched_pattern=matched_pattern,
litellm_deployment_litellm_model=deployment["litellm_params"]["model"],
)
if "*" in deployment["litellm_params"]["model"] and isinstance(new_deployment.get("model_info"), dict):
left_to_the_catalog = _cache_flags_left_to_the_catalog(deployment["litellm_params"])
new_deployment["model_info"] = {
key: value for key, value in new_deployment["model_info"].items() if key not in left_to_the_catalog
}
new_deployments.append(new_deployment)
return new_deployments

50
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -21,12 +21,14 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.batch_utils import InMemoryFile
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAudioObject,
ChatCompletionCachedContent,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionPromptCacheBreakpoint,
@ -2183,6 +2185,9 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = (
"set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)"
)
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
@ -2262,6 +2267,51 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
)
def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT:
"""OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block.
A ``cache_control`` the client already wrote on the block wins, TTL included."""
marker: Final = block.get("prompt_cache_breakpoint")
if marker is None:
return block
unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
control: Final = (
None
if block.get("cache_control") is not None
else _cache_control_for_prompt_cache_breakpoint(marker, drop_params)
)
marked: Final = unmarked if control is None else {**unmarked, "cache_control": control}
return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed
def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None:
try:
breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
except ValidationError:
if not drop_params:
raise UnsupportedParamsError(
message=(
f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got "
f"{marker!r}. Fix the marker, or "
f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
return None
ttl: Final = breakpoint_mark.get("ttl")
if ttl is not None and not drop_params:
raise UnsupportedParamsError(
message=(
f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint "
f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send "
"a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block "
"where the model offers a 1h cache."
),
status_code=400,
)
return ChatCompletionCachedContent(type="ephemeral")
_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"})
_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])

100
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -77,7 +77,7 @@ from .common_utils import (
parse_tool_call_arguments,
salvage_concatenated_tool_arguments,
)
from .image_handling import convert_url_to_base64
from .image_handling import convert_url_to_base64, is_pdf_media_type
def default_pt(messages):
@ -911,8 +911,17 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) ->
)
def _anthropic_https_url_block(
image_url: str, image_format: str | None
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
source: Final = AnthropicContentParamSourceUrl(type="url", url=image_url)
if image_format and is_pdf_media_type(image_format):
return AnthropicMessagesDocumentParam(type="document", source=source)
return AnthropicMessagesImageParam(type="image", source=source)
def create_anthropic_image_param(
image_url_input: str | dict,
image_url_input: str | Mapping[str, object],
format: str | None = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
@ -921,13 +930,11 @@ def create_anthropic_image_param(
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
# Extract URL and format from input
if isinstance(image_url_input, str):
image_url = image_url_input
else:
image_url = image_url_input.get("url", "")
if format is None:
format = image_url_input.get("format")
url_value: Final = image_url_input if isinstance(image_url_input, str) else image_url_input.get("url")
format_value: Final = None if isinstance(image_url_input, str) else image_url_input.get("format")
image_url: Final = url_value if isinstance(url_value, str) else ""
explicit_format: Final = format if format is not None else format_value
image_format: Final = explicit_format if isinstance(explicit_format, str) else None
# Check if the image URL is an HTTP/HTTPS URL
if image_url.startswith("http://") or image_url.startswith("https://"):
@ -935,21 +942,14 @@ def create_anthropic_image_param(
# as these providers don't support URL sources for images
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url: Final = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format)
return _anthropic_content_element_factory(image_chunk)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=image_format)
return anthropic_content_element_factory(image_chunk)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSourceUrl(
type="url",
url=image_url,
),
)
return _anthropic_https_url_block(image_url, image_format)
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format)
return _anthropic_content_element_factory(image_chunk)
image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=image_format)
return anthropic_content_element_factory(image_chunk)
# The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex
@ -2255,16 +2255,16 @@ def add_cache_control_to_content(
return anthropic_content_element
def _anthropic_content_element_factory(
def anthropic_content_element_factory(
image_chunk: GenericImageParsingChunk,
) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam:
if image_chunk["media_type"] == "application/pdf":
if is_pdf_media_type(image_chunk["media_type"]):
_anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = (
AnthropicMessagesDocumentParam(
type="document",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
media_type="application/pdf",
data=image_chunk["data"],
),
)
@ -4254,6 +4254,7 @@ def _convert_to_bedrock_tool_call_invoke(
tool_calls: list,
model: str | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> list[BedrockContentBlock]:
"""
OpenAI tool invokes:
@ -4338,6 +4339,7 @@ def _convert_to_bedrock_tool_call_invoke(
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if _cache_point_block is not None:
_parts_list.append(_cache_point_block)
@ -4358,6 +4360,7 @@ def _convert_to_bedrock_tool_call_invoke(
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if cache_point_block is not None:
_parts_list.append(cache_point_block)
@ -5029,6 +5032,7 @@ class BedrockConverseMessagesProcessor:
deployment_model: str | None = None,
base_model: str | None = None,
drop_params: bool = False,
model_info: Mapping[str, object] | None = None,
) -> list[BedrockMessageBlock]:
from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking
@ -5091,6 +5095,7 @@ class BedrockConverseMessagesProcessor:
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
@ -5098,7 +5103,11 @@ class BedrockConverseMessagesProcessor:
elif message_content and isinstance(message_content, str):
_part = BedrockContentBlock(text=message_content)
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
message_block, block_type="content_block", model=model, deployment_model=deployment_model
message_block,
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
user_content.append(_part)
if _cache_point_block is not None:
@ -5149,6 +5158,7 @@ class BedrockConverseMessagesProcessor:
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if cache_point_block is not None:
tool_content.append(cache_point_block)
@ -5214,6 +5224,7 @@ class BedrockConverseMessagesProcessor:
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if _cache_point_block is not None and not (
drop_params and _as_thinking_block(element) is not None
@ -5231,6 +5242,7 @@ class BedrockConverseMessagesProcessor:
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
@ -5239,7 +5251,7 @@ class BedrockConverseMessagesProcessor:
if _tool_calls:
assistant_content.extend(
_convert_to_bedrock_tool_call_invoke(
_tool_calls, model=model, deployment_model=deployment_model
_tool_calls, model=model, deployment_model=deployment_model, model_info=model_info
)
)
@ -5434,6 +5446,7 @@ def _bedrock_converse_messages_pt(
deployment_model: str | None = None,
base_model: str | None = None,
drop_params: bool = False,
model_info: Mapping[str, object] | None = None,
) -> list[BedrockMessageBlock]:
"""
Converts given messages from OpenAI format to Bedrock format
@ -5502,6 +5515,7 @@ def _bedrock_converse_messages_pt(
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if _cache_point_block is not None:
_parts.append(_cache_point_block)
@ -5509,7 +5523,11 @@ def _bedrock_converse_messages_pt(
elif message_content and isinstance(message_content, str):
_part = BedrockContentBlock(text=message_content)
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
message_block, block_type="content_block", model=model, deployment_model=deployment_model
message_block,
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
user_content.append(_part)
if _cache_point_block is not None:
@ -5562,6 +5580,7 @@ def _bedrock_converse_messages_pt(
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if cache_point_block is not None:
tool_content.append(cache_point_block)
@ -5628,6 +5647,7 @@ def _bedrock_converse_messages_pt(
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if _cache_point_block is not None and not (
drop_params and _as_thinking_block(element) is not None
@ -5640,14 +5660,20 @@ def _bedrock_converse_messages_pt(
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
assistant_message_block, block_type="content_block", model=model, deployment_model=deployment_model
assistant_message_block,
block_type="content_block",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if _cache_point_block is not None:
assistant_content.append(_cache_point_block)
_tool_calls = assistant_message_block.get("tool_calls", [])
if _tool_calls:
assistant_content.extend(
_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model, deployment_model=deployment_model)
_convert_to_bedrock_tool_call_invoke(
_tool_calls, model=model, deployment_model=deployment_model, model_info=model_info
)
)
msg_i += 1
@ -5703,7 +5729,10 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
def add_cache_point_tool_block(
tool: dict, model: str | None = None, deployment_model: str | None = None
tool: dict,
model: str | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> BedrockToolBlock | None:
from litellm.llms.bedrock.common_utils import (
bedrock_cache_point_ttl_supported,
@ -5711,7 +5740,7 @@ def add_cache_point_tool_block(
)
cache_control: Final = tool.get("cache_control", None)
if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model):
if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model, model_info):
cache_point: Final = cache_control.get("type", "ephemeral")
if cache_point == "ephemeral":
cache_point_block: Final[CachePointBlock] = {"type": "default"}
@ -5748,7 +5777,10 @@ def _is_bedrock_tool_block(tool: dict) -> bool:
def _bedrock_tools_pt(
tools: list, model: str | None = None, deployment_model: str | None = None
tools: list,
model: str | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> list[BedrockToolBlock]:
"""
OpenAI tools looks like:
@ -5869,7 +5901,9 @@ def _bedrock_tools_pt(
tool_block_list.append(tool_block)
## ADD CACHE POINT TOOL BLOCK ##
cache_point_tool_block = add_cache_point_tool_block(tool, model=model, deployment_model=deployment_model)
cache_point_tool_block = add_cache_point_tool_block(
tool, model=model, deployment_model=deployment_model, model_info=model_info
)
if cache_point_tool_block is not None:
tool_block_list.append(cache_point_tool_block)

263
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -5,28 +5,33 @@ Helper functions to handle images passed in messages
import asyncio
import base64
import time
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from itertools import chain
from functools import partial
from itertools import accumulate, chain, takewhile, tee
from types import MappingProxyType
from typing import Final
from typing import Final, TypeVar
from urllib.parse import urlparse
import httpx
from httpx import Response
from httpx import USE_CLIENT_DEFAULT, Response
import litellm
from litellm import verbose_logger
from litellm.caching.caching import InMemoryCache
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT
from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
from litellm.types.llms.openai import AllMessageValues
MAX_IMGS_IN_MEMORY: Final = 10
MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20
_Fetched = TypeVar("_Fetched")
_FETCH_ATTEMPTS: Final = 3
in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY)
pdf_probe_verdicts: Final = InMemoryCache()
def _header_value(response: Response, name: str) -> str | None:
@ -209,6 +214,173 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool:
return media.part_type == "image_url"
_PDF_MEDIA_TYPES: Final = frozenset({"application/pdf", "application/x-pdf"})
_GENERIC_MEDIA_TYPES: Final = frozenset({"application/octet-stream", "binary/octet-stream"})
_PDF_SIGNATURE: Final = b"%PDF-"
_PDF_SIGNATURE_WINDOW: Final = 1024
_PDF_PROBE_HEADERS: Final = MappingProxyType({"Range": f"bytes=0-{_PDF_SIGNATURE_WINDOW - 1}"})
_PDF_PROBE_STATUSES: Final = frozenset({200, 206})
_PDF_PROBE_FAILURES: Final = (httpx.HTTPError, httpx.InvalidURL, ValueError)
_ASYNC_PDF_PROBE_FAILURES: Final = (*_PDF_PROBE_FAILURES, asyncio.TimeoutError)
_PDF_PROBE_VERDICT_TTL: Final = 600
_PDF_PROBE_FAILURE_TTL: Final = 60
_DECODED_BODY_HEADERS: Final = frozenset({"content-encoding", "content-length", "transfer-encoding"})
def _bare_media_type(media_type: str) -> str:
return media_type.split(";", 1)[0].strip().lower()
def is_pdf_media_type(media_type: str) -> bool:
return _bare_media_type(media_type) in _PDF_MEDIA_TYPES
def is_unformatted_pdf_link(url: str, image_format: str | None) -> bool:
lowered: Final = url.lower()
return not image_format and (urlparse(lowered).path.endswith(".pdf") or lowered.endswith(".pdf"))
def is_unformatted_pdf_link_image(media: RemoteMedia) -> bool:
image_format: Final = media.fields.get("format")
return inline_remote_image_urls(media) and is_unformatted_pdf_link(
media.url, image_format if isinstance(image_format, str) else None
)
def _serves_pdf(response: Response) -> bool:
if response.status_code not in _PDF_PROBE_STATUSES:
return False
served_type: Final = _bare_media_type(_header_value(response, "Content-Type") or "")
if served_type and served_type not in _GENERIC_MEDIA_TYPES:
return served_type in _PDF_MEDIA_TYPES
return _PDF_SIGNATURE in response.content[:_PDF_SIGNATURE_WINDOW]
def _signature_window_chunks(chunks: Iterable[bytes], deadline: float) -> Iterator[bytes]:
reading, counting = tee(chunks)
read_so_far: Final = chain((0,), accumulate(map(len, counting)))
still_reading: Final = takewhile(
lambda total: total < _PDF_SIGNATURE_WINDOW and time.monotonic() < deadline, read_so_far
)
return (chunk for _, chunk in zip(still_reading, reading))
def _with_body_prefix(response: Response, prefix: bytes) -> Response:
return Response(
response.status_code,
headers=[(name, value) for name, value in response.headers.multi_items() if name not in _DECODED_BODY_HEADERS],
content=prefix,
request=response.request,
)
@dataclass(frozen=True, slots=True)
class _BodyPrefixReader:
client: httpx.Client
deadline: float
def get(
self,
url: str,
*,
headers: Mapping[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
) -> Response:
with self.client.stream(
"GET",
url,
headers=headers,
follow_redirects=follow_redirects,
timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
) as response:
prefix: Final = b"".join(_signature_window_chunks(response.iter_bytes(), self.deadline))
return _with_body_prefix(response, prefix[:_PDF_SIGNATURE_WINDOW])
@dataclass(frozen=True, slots=True)
class _AsyncBodyPrefixReader:
client: httpx.AsyncClient
async def get(
self,
url: str,
*,
headers: Mapping[str, str] | None = None,
follow_redirects: bool = False,
timeout: float | httpx.Timeout | None = None,
max_response_bytes: int | None = None,
) -> Response:
async with self.client.stream(
"GET",
url,
headers=headers,
follow_redirects=follow_redirects,
timeout=USE_CLIENT_DEFAULT if timeout is None else timeout,
) as response:
return _with_body_prefix(response, await anext(response.aiter_bytes(max_response_bytes), b""))
def _cached_pdf_verdict(url: str) -> bool | None:
cached: Final = pdf_probe_verdicts.get_cache(url)
return cached if isinstance(cached, bool) else None
def _remember_pdf_verdict(url: str, verdict: bool, ttl: int) -> bool:
pdf_probe_verdicts.set_cache(url, verdict, ttl=ttl)
return verdict
def _remember_probe_response(url: str, response: Response) -> bool:
ttl: Final = _PDF_PROBE_VERDICT_TTL if response.status_code in _PDF_PROBE_STATUSES else _PDF_PROBE_FAILURE_TTL
return _remember_pdf_verdict(url, _serves_pdf(response), ttl)
def _failed_pdf_probe(url: str, error: Exception) -> bool:
verbose_logger.debug("PDF probe of %s failed, sending it as an image link: %s", url, error)
return _remember_pdf_verdict(url, False, _PDF_PROBE_FAILURE_TTL)
def serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
return False
cached: Final = _cached_pdf_verdict(url)
if cached is not None:
return cached
try:
response: Final = safe_get(
_BodyPrefixReader(litellm.module_level_client.client, deadline=time.monotonic() + timeout),
url,
headers=dict(_PDF_PROBE_HEADERS),
timeout=timeout,
)
except _PDF_PROBE_FAILURES as e:
return _failed_pdf_probe(url, e)
return _remember_probe_response(url, response)
async def async_serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool:
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
return False
cached: Final = _cached_pdf_verdict(url)
if cached is not None:
return cached
try:
response: Final = await asyncio.wait_for(
async_safe_get(
_AsyncBodyPrefixReader(litellm.module_level_aclient.client),
url,
headers=dict(_PDF_PROBE_HEADERS),
timeout=timeout,
max_response_bytes=_PDF_SIGNATURE_WINDOW,
),
timeout,
)
except _ASYNC_PDF_PROBE_FAILURES as e:
return _failed_pdf_probe(url, e)
return _remember_probe_response(url, response)
def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None:
if fields.get("type") != "image_url":
return None
@ -292,15 +464,40 @@ def _inline_part(part: object, data_urls: Mapping[str, str], should_inline: Call
return _inline(remote, data_url) if data_url is not None else part
def _inline_message(
message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
) -> AllMessageValues:
def _with_parts(message: AllMessageValues, rewrite: Callable[[object], object]) -> AllMessageValues:
parts: Final = _content_parts(message)
if not parts:
return message
inlined_parts: Final = [_inline_part(part, data_urls, should_inline) for part in parts]
inlined_message: Final = {**message, "content": inlined_parts}
return inlined_message # pyright: ignore[reportReturnType] # the same message with its remote parts inlined
rewritten: Final = {**message, "content": [rewrite(part) for part in parts]}
return rewritten # pyright: ignore[reportReturnType] # the same message with some of its parts rewritten
def _inline_message(
message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool]
) -> AllMessageValues:
return _with_parts(message, partial(_inline_part, data_urls=data_urls, should_inline=should_inline))
def _pdf_formatted_part(part: object, pdf_links: frozenset[str], should_probe: Callable[[RemoteMedia], bool]) -> object:
remote: Final = _parse_remote_part(part)
match remote:
case _RemoteImage(fields, image_url, url) if url in pdf_links and should_probe(_remote_media(remote)):
image_url_fields: Final = image_url if image_url is not None else _NO_FIELDS
return {**fields, "image_url": {**image_url_fields, "url": url, "format": "application/pdf"}}
case _:
return part
def _with_pdf_formats(
messages: Iterable[AllMessageValues],
probed: Iterable[tuple[str, bool]],
should_probe: Callable[[RemoteMedia], bool],
) -> tuple[AllMessageValues, ...]:
pdf_links: Final = frozenset(url for url, is_pdf in probed if is_pdf)
if not pdf_links:
return tuple(messages)
rewrite: Final = partial(_pdf_formatted_part, pdf_links=pdf_links, should_probe=should_probe)
return tuple(_with_parts(message, rewrite) for message in messages)
async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str:
@ -308,11 +505,12 @@ async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float
return await async_convert_url_to_base64(url, timeout=timeout)
async def async_convert_urls_to_base64(
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
) -> tuple[str, ...]:
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
async def _serves_pdf_within(url: str, in_flight: asyncio.Semaphore) -> bool:
async with in_flight:
return await async_serves_pdf(url)
async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) -> tuple[_Fetched, ...]:
try:
return tuple(await asyncio.gather(*fetches))
except BaseException:
@ -322,6 +520,15 @@ async def async_convert_urls_to_base64(
raise
async def async_convert_urls_to_base64(
remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None
) -> tuple[str, ...]:
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
return await _gather_or_cancel_all(
tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls)
)
def _remote_urls_to_inline(
messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool]
) -> tuple[str, ...]:
@ -351,3 +558,27 @@ async def async_inline_remote_media(
data_urls: Final = await async_convert_urls_to_base64(remote_urls)
inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True)))
return [_inline_message(message, inlined, should_inline) for message in messages]
def format_pdf_links(
messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
) -> tuple[AllMessageValues, ...]:
links: Final = _remote_urls_to_inline(messages, should_probe)
if not links:
return tuple(messages)
with ThreadPoolExecutor(max_workers=min(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, len(links))) as probes:
verdicts: Final = tuple(probes.map(serves_pdf, links))
return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)
async def async_format_pdf_links(
messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool]
) -> tuple[AllMessageValues, ...]:
links: Final = _remote_urls_to_inline(messages, should_probe)
if not links:
return tuple(messages)
in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES)
verdicts: Final = await _gather_or_cancel_all(
tuple(asyncio.create_task(_serves_pdf_within(url, in_flight)) for url in links)
)
return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe)

738
litellm/litellm_core_utils/realtime_streaming.py
View file

File diff suppressed because it is too large Load diff

54
litellm/litellm_core_utils/redact_messages.py
View file

@ -10,7 +10,7 @@
import asyncio
import copy
import inspect
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, Any, Final
@ -107,7 +107,13 @@ def redacted_response_obj_for_custom_logger(response_obj: object, custom_logger:
def _is_redactable_response(response_obj: object) -> bool:
return isinstance(
response_obj, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse)
response_obj,
(
litellm.ModelResponse,
litellm.TextCompletionResponse,
litellm.ResponsesAPIResponse,
litellm.EmbeddingResponse,
),
) or (isinstance(response_obj, dict) and ("choices" in response_obj or "output" in response_obj))
@ -171,6 +177,8 @@ def _redact_streaming_response(streaming_response):
_redact_bedrock_guardrail_trace(streaming_response)
elif isinstance(streaming_response, litellm.ResponsesAPIResponse):
return perform_redaction({}, streaming_response)
elif isinstance(streaming_response, litellm.TranscriptionResponse):
streaming_response.text = REDACTED_BY_LITELLM
elif hasattr(streaming_response, "output"):
_redact_responses_api_output(streaming_response.output)
if hasattr(streaming_response, "reasoning") and streaming_response.reasoning is not None:
@ -206,6 +214,7 @@ def _redact_choice_content(choice):
choice.message.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.message, "thinking_blocks"):
choice.message.thinking_blocks = None
_drop_field_when_present(choice.message, "reasoning_items")
_redact_provider_specific_fields(choice.message)
_redact_tool_calls(getattr(choice.message, "tool_calls", None))
_redact_function_call(getattr(choice.message, "function_call", None))
@ -216,12 +225,14 @@ def _redact_choice_content(choice):
choice.delta.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "thinking_blocks"):
choice.delta.thinking_blocks = None
_drop_field_when_present(choice.delta, "reasoning_items")
_redact_provider_specific_fields(choice.delta)
_redact_tool_calls(getattr(choice.delta, "tool_calls", None))
_redact_function_call(getattr(choice.delta, "function_call", None))
elif isinstance(choice, litellm.utils.TextChoices):
_redact_field_when_present(choice, "text")
_redact_field_when_present(choice, "reasoning_content")
_drop_field_when_present(choice, "logprobs")
def _as_mapping(value: object) -> Mapping[str, object] | None:
@ -282,6 +293,11 @@ def _redact_field_when_present(item: object, field: str) -> None:
setattr(item, field, REDACTED_BY_LITELLM)
def _drop_field_when_present(item: object, field: str) -> None:
if getattr(item, field, None) is not None:
setattr(item, field, None)
def _redact_responses_api_output(output_items):
"""Helper to redact ResponsesAPIResponse output items."""
for output_item in output_items:
@ -438,32 +454,22 @@ def _redact_tool_calls_dict(message: Mapping[str, object]) -> None:
function_call["arguments"] = REDACTED_BY_LITELLM
def _redact_model_response_dict_choices(choices, redacted_str: str):
def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str: str) -> None:
for choice in choices:
if isinstance(choice, dict):
_redact_provider_specific_fields_dict(choice)
if choice.get("token_ids") is not None:
choice["token_ids"] = None
if "message" in choice and isinstance(choice["message"], dict):
if choice["message"].get("content") is not None:
choice["message"]["content"] = redacted_str
if choice["message"].get("reasoning_content") is not None:
choice["message"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
_redact_tool_calls_dict(choice["message"])
elif "delta" in choice and isinstance(choice["delta"], dict):
if choice["delta"].get("content") is not None:
choice["delta"]["content"] = redacted_str
if choice["delta"].get("reasoning_content") is not None:
choice["delta"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_tool_calls_dict(choice["delta"])
if (
body := next((choice[key] for key in ("message", "delta") if isinstance(choice.get(key), dict)), None)
) is not None:
for field in ("content", "reasoning_content"):
if body.get(field) is not None:
body[field] = redacted_str
for field in ("thinking_blocks", "reasoning_items", "audio"):
if field in body:
body[field] = None
_redact_tool_calls_dict(body)
elif choice.get("text") is not None:
choice["text"] = redacted_str
else:
@ -586,7 +592,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
return {"text": REDACTED_BY_LITELLM}
_result: Final = copy.deepcopy(result)
if isinstance(_result, litellm.ModelResponse):
if isinstance(_result, (litellm.ModelResponse, litellm.TextCompletionResponse)):
if hasattr(_result, "choices") and _result.choices is not None:
for choice in _result.choices:
_redact_choice_content(choice)

6
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -34,6 +34,7 @@ from litellm.llms.anthropic.messages.adapters.transformation import (
)
from litellm.llms.anthropic.messages.convertible_blocks import (
keep_non_string_tool_result_texts,
keep_url_documents,
scannable_tool_result_request,
)
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
@ -1187,7 +1188,10 @@ class AnthropicMessagesHandler(BaseTranslation):
cast(Sequence[object], converted), # cast-ok: Anthropic message payload
original_tool_names,
)
data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages"))
original_messages: Final = data.get("messages")
data["messages"] = keep_url_documents(
keep_non_string_tool_result_texts(converted, original_messages), original_messages
)
@staticmethod
def _extract_midturn_system_text(

35
litellm/llms/anthropic/chat/transformation.py
View file

@ -27,6 +27,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.litellm_core_utils.asyncify import is_event_loop_running
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
@ -47,8 +48,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
RemoteMedia,
async_format_pdf_links,
async_inline_remote_media,
format_pdf_links,
inline_remote_image_urls,
is_unformatted_pdf_link_image,
)
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
message_field,
@ -2333,6 +2337,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def inlines_remote_media(self, media: RemoteMedia) -> bool:
return inline_remote_image_urls(media) and media.url.startswith("http://")
def probes_pdf_link(self, media: RemoteMedia) -> bool:
return is_unformatted_pdf_link_image(media) and not self.inlines_remote_media(media)
def _with_pdf_links_formatted(self, messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]:
return messages if is_event_loop_running() else format_pdf_links(messages, should_probe=self.probes_pdf_link)
async def async_transform_request(
self,
model: str,
@ -2341,9 +2351,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
litellm_params: dict[str, object], # mutable-ok: BaseConfig signature
headers: dict[str, object], # mutable-ok: BaseConfig signature
) -> dict[str, object]: # mutable-ok: BaseConfig signature
inlined: Final = await async_inline_remote_media(messages, should_inline=self.inlines_remote_media)
return self.transform_request(
model=model,
messages=await async_inline_remote_media(messages, should_inline=self.inlines_remote_media),
messages=list(await async_format_pdf_links(inlined, should_probe=self.probes_pdf_link)),
optional_params=optional_params,
litellm_params=litellm_params,
headers=headers,
@ -2359,6 +2370,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def translates_prompt_cache_breakpoint(self, model: str) -> bool:
return True
def transform_request(
self,
model: str,
@ -2372,19 +2386,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""
## VALIDATE REQUEST
"""Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed)."""
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, cache_control_count
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
)
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
translated_messages: Final = (
list(
AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
messages,
drop_params,
cache_control_count,
lambda: AnthropicCacheControlHook.count_chat_external_cache_breakpoints(optional_params),
)
)
if self.translates_prompt_cache_breakpoint(model)
else messages
)
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
translated_messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
else translated_messages
)
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
@ -2477,7 +2504,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
try:
anthropic_messages = anthropic_messages_pt(
model=model,
messages=list(conversation),
messages=list(self._with_pdf_links_formatted(conversation)),
llm_provider=self._resolved_provider,
drop_params=drop_params,
)

26
litellm/llms/anthropic/common_utils.py
View file

@ -28,7 +28,7 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
@ -307,9 +307,10 @@ def count_tokens_feature_betas(
)
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
def supports_anthropic_cache_control(
model: str, custom_llm_provider: str | None, model_info: Mapping[str, object] | None = None
) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching
try:
provider: Final = custom_llm_provider if custom_llm_provider is not None else get_llm_provider(model=model)[1]
@ -318,10 +319,27 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None
return (
provider in ("anthropic", "bedrock", "vertex_ai", "azure_ai")
and "claude" in model.lower()
and supports_prompt_caching(model=model, custom_llm_provider=provider)
and _deployment_supports_prompt_caching(model, provider, model_info)
)
def _deployment_supports_prompt_caching(model: str, provider: str, model_info: Mapping[str, object] | None) -> bool:
from litellm.utils import catalog_with_runtime_registrations, get_model_info, supports_prompt_caching
declared: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching"))
if declared is not None:
return declared
live: Final = supports_prompt_caching(model=model, custom_llm_provider=provider)
if model_info is None:
return live
try:
key: Final = get_model_info(model=model, custom_llm_provider=provider).get("key")
except Exception: # noqa: BLE001 # an unmapped model keeps the live answer
return live
catalog_entry: Final = catalog_with_runtime_registrations().get(key)
return live if catalog_entry is None else catalog_entry.get("supports_prompt_caching") is True
def _tool_config_point_control(points: object) -> Mapping[object, object] | None:
point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ())
tool_config_point: Final = next(

7
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -49,7 +49,10 @@ _PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"})
def _count_image(block: JsonValue) -> JsonValue:
if not isinstance(block, dict) or block.get("type") != "image_url":
return block
from litellm.litellm_core_utils.prompt_templates.factory import convert_to_anthropic_image_obj
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_content_element_factory,
convert_to_anthropic_image_obj,
)
image_block: Final = _IMAGE_BLOCK.validate_python(block)
image_url: Final = image_block["image_url"]
@ -57,7 +60,7 @@ def _count_image(block: JsonValue) -> JsonValue:
openai_image_url=image_url if isinstance(image_url, str) else image_url["url"],
format=image_url.get("format") if isinstance(image_url, dict) else None,
)
image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source})
image: Final = _COUNT_REQUEST.validate_python(anthropic_content_element_factory(source))
return {**{key: value for key, value in block.items() if key not in {"type", "image_url"}}, **image}

70
litellm/llms/anthropic/messages/convertible_blocks.py
View file

@ -314,6 +314,76 @@ def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: obj
return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten]
def _url_source_url(block: object) -> str | None:
match as_str_mapping(block):
case {"type": "image" | "document", "source": {"type": "url", "url": str(url)}}:
return url
case _:
return None
def _is_document(block: object) -> bool:
match as_str_mapping(block):
case {"type": "document"}:
return True
case _:
return False
@dataclass(frozen=True, slots=True, order=True)
class _UrlBlockPlace:
in_tool_result: bool
url: str
def _url_sourced_blocks(block: object, in_tool_result: bool = False) -> tuple[tuple[_UrlBlockPlace, object], ...]:
url: Final = _url_source_url(block)
if url is not None:
return ((_UrlBlockPlace(in_tool_result, url), block),)
match as_str_mapping(block):
case {"type": "tool_result", "content": list() as content}:
in_this_tool_result: Final = partial(_url_sourced_blocks, in_tool_result=True)
return tuple(chain.from_iterable(map(in_this_tool_result, _list_items(content))))
case _:
return ()
def _entry_place(entry: tuple[_UrlBlockPlace, object]) -> _UrlBlockPlace:
return entry[0]
def _entry_block(entry: tuple[_UrlBlockPlace, object]) -> object:
return entry[1]
def _restored_url_document(
block: object, originals: Mapping[_UrlBlockPlace, Iterator[object]], in_tool_result: bool = False
) -> object:
url: Final = _url_source_url(block)
if url is not None:
remaining: Final = originals.get(_UrlBlockPlace(in_tool_result, url))
original: Final = None if remaining is None else next(remaining, None)
return original if _is_document(original) else block
match as_str_mapping(block):
case {"type": "tool_result", "content": list() as content} as tool_result:
restore: Final = partial(_restored_url_document, originals=originals, in_tool_result=True)
return {**tool_result, "content": list(map(restore, _list_items(content)))}
case _:
return block
def keep_url_documents(rewritten: Sequence[object], original: object) -> Sequence[object]:
blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original)))
by_place: Final = tuple(sorted(chain.from_iterable(map(_url_sourced_blocks, blocks)), key=_entry_place))
if not any(_is_document(block) for _, block in by_place):
return rewritten
originals_in_order: Final = MappingProxyType(
{place: iter(tuple(map(_entry_block, group))) for place, group in groupby(by_place, key=_entry_place)}
)
restore: Final = partial(_restored_url_document, originals=originals_in_order)
return [_with_user_blocks(message, restore) for message in rewritten]
def _without_non_string_text(block: object) -> object:
match as_str_mapping(block):
case {"type": "tool_result", "content": list()} as tool_result:

7
litellm/llms/anthropic/messages/responses_adapters/handler.py
View file

@ -13,6 +13,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
routed_model_info,
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.asyncify import (
@ -187,6 +188,7 @@ def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, ob
provider if isinstance(provider, str) else None,
forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
forwarded_kwargs.get("prompt_cache_options"),
routed_model_info(forwarded_kwargs.get("model_info")),
)
@ -194,10 +196,11 @@ def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[s
provider: Final = forwarded_kwargs.get("custom_llm_provider")
provider_name: Final = provider if isinstance(provider, str) else None
base_model: Final = forwarded_kwargs.get("base_model")
return supports_openai_prompt_cache_breakpoint(model, provider_name) or (
model_info: Final = routed_model_info(forwarded_kwargs.get("model_info"))
return supports_openai_prompt_cache_breakpoint(model, provider_name, model_info) or (
isinstance(base_model, str)
and bool(base_model)
and supports_openai_prompt_cache_breakpoint(base_model, provider_name)
and supports_openai_prompt_cache_breakpoint(base_model, provider_name, model_info)
)

41
litellm/llms/azure/audio_transcriptions.py
View file

@ -1,11 +1,17 @@
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import TYPE_CHECKING, Final
from openai import AsyncAzureOpenAI, AzureOpenAI
from openai import AsyncAzureOpenAI, AsyncStream, AzureOpenAI, Stream
from openai.types.audio import TranscriptionStreamEvent
from pydantic import BaseModel
from litellm._uuid import uuid
from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name
from litellm.llms.base_llm.audio_transcription.transformation import (
acreate_sdk_transcription,
create_sdk_transcription,
sdk_compatible_transcription_request_data,
)
from litellm.types.utils import FileTypes
from litellm.utils import (
TranscriptionResponse,
@ -38,13 +44,18 @@ class AzureAudioTranscription(AzureChatCompletion):
atranscription: bool = False,
litellm_params: dict | None = None,
custom_llm_provider: str = "azure",
) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
) -> (
TranscriptionResponse
| Stream[TranscriptionStreamEvent]
| Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
):
data: Final = {"model": model, "file": audio_file, **optional_params}
sdk_data: Final = sdk_compatible_transcription_request_data(data)
if atranscription is True:
return self.async_audio_transcriptions(
audio_file=audio_file,
data=data,
data=sdk_data,
model_response=model_response,
timeout=timeout,
api_key=api_key,
@ -58,7 +69,7 @@ class AzureAudioTranscription(AzureChatCompletion):
)
azure_client: Final = self.get_azure_openai_client(
api_version=api_version,
api_version=None if self.is_azure_v1_api_version(api_version) else api_version,
api_base=api_base,
api_key=api_key,
model=model,
@ -84,10 +95,9 @@ class AzureAudioTranscription(AzureChatCompletion):
},
)
response: Final = azure_client.audio.transcriptions.create(
**data,
timeout=timeout,
)
_, response = create_sdk_transcription(azure_client, sdk_data, timeout)
if isinstance(response, Stream):
return response
if isinstance(response, BaseModel):
stringified_response = response.model_dump()
@ -114,7 +124,7 @@ class AzureAudioTranscription(AzureChatCompletion):
self,
audio_file: FileTypes,
model: str,
data: dict,
data: Mapping[str, object],
model_response: TranscriptionResponse,
timeout: float,
logging_obj: "LiteLLMLoggingObj",
@ -125,7 +135,7 @@ class AzureAudioTranscription(AzureChatCompletion):
max_retries=None,
litellm_params: dict | None = None,
custom_llm_provider: str = "azure",
) -> TranscriptionResponse:
) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]:
response = None
try:
async_azure_client: Final = self.get_azure_openai_client(
@ -155,12 +165,9 @@ class AzureAudioTranscription(AzureChatCompletion):
},
)
raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create(
**data, timeout=timeout
)
headers: Final = dict(raw_response.headers)
response = raw_response.parse()
headers, response = await acreate_sdk_transcription(async_azure_client, data, timeout)
if isinstance(response, AsyncStream):
return response
if isinstance(response, BaseModel):
stringified_response = response.model_dump()

96
litellm/llms/azure/realtime/handler.py
View file

@ -5,9 +5,11 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, Protocol, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.url_utils import join_api_base
@ -48,17 +50,55 @@ async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTe
pass
def azure_realtime_requires_ga(model: str) -> bool:
try:
azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure")
except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly
try:
openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai")
except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly
return False
openai_entry: Final = openai_model_info.get("provider_specific_entry")
return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1
azure_entry: Final = azure_model_info.get("provider_specific_entry")
return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1
def _is_ga_protocol(protocol: str | None) -> bool:
return protocol is not None and protocol.upper() in ("GA", "V1")
@dataclass(frozen=True, slots=True)
class AzureRealtimeProtocol:
name: str
bridges_beta_client_to_ga: bool = False
def azure_realtime_protocol_for_client(
configured_protocol: object,
*,
model: str,
base_model: str | None = None,
realtime_mode: str,
query_params: RealtimeQueryParams | None,
websocket: ScopedWebSocket,
) -> str:
) -> AzureRealtimeProtocol:
serves_a_ga_only_session: Final = (
realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription"
)
client_wants_beta: Final = client_sent_openai_beta_realtime_header(websocket)
if isinstance(configured_protocol, str) and configured_protocol:
return configured_protocol
if (query_params or {}).get("intent") == "transcription":
return "GA"
return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA"
return AzureRealtimeProtocol(
name=configured_protocol,
bridges_beta_client_to_ga=client_wants_beta
and not serves_a_ga_only_session
and _is_ga_protocol(configured_protocol),
)
if serves_a_ga_only_session:
return AzureRealtimeProtocol(name="GA")
if azure_realtime_requires_ga(base_model or model):
return AzureRealtimeProtocol(name="GA", bridges_beta_client_to_ga=client_wants_beta)
return AzureRealtimeProtocol(name="beta" if client_wants_beta else "GA")
class _ProxyClientWebSocket(Protocol):
@ -92,6 +132,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
api_version: str | None,
realtime_protocol: str | None = None,
query_params: RealtimeQueryParams | None = None,
realtime_mode: str = "realtime",
) -> str:
"""
Construct Azure realtime WebSocket URL.
@ -104,6 +145,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
- "GA" or "v1": Uses /openai/v1/realtime (GA path)
- "beta" or None: Uses /openai/realtime (beta path, default)
query_params: Extra query params to forward (e.g. intent=transcription).
realtime_mode: Realtime surface to target, "realtime" or "translation".
Returns:
WebSocket URL string
@ -117,24 +159,29 @@ class AzureOpenAIRealtime(AzureChatCompletion):
api_base = api_base.replace("https://", "wss://")
# Determine path based on realtime_protocol (case-insensitive)
_is_ga: Final = realtime_protocol is not None and realtime_protocol.upper() in (
"GA",
"V1",
)
_is_ga: Final = _is_ga_protocol(realtime_protocol)
intent: Final = (query_params or {}).get("intent")
if _is_ga:
path = "/openai/v1/realtime"
query_parts = []
if intent != "transcription" and (query_params is None or "model" in query_params):
query_parts.append(urlencode({"model": model}))
else:
# Default to beta path for backwards compatibility
path = "/openai/realtime"
query_parts = [urlencode({"api-version": api_version, "deployment": model})]
path: Final = (
"/openai/v1/realtime/translations"
if realtime_mode == "translation"
else "/openai/v1/realtime"
if _is_ga
else "/openai/realtime"
)
base_query_parts: Final = (
(urlencode((("model", model),)),)
if realtime_mode == "translation"
else (
(urlencode((("model", model),)),)
if intent != "transcription" and (query_params is None or "model" in query_params)
else ()
)
if _is_ga
else (urlencode((("api-version", api_version), ("deployment", model))),)
)
if intent:
query_parts.append(urlencode({"intent": intent}))
query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts
qs: Final = "&".join(query_parts)
endpoint_url: Final = join_api_base(api_base, path)
@ -165,13 +212,16 @@ class AzureOpenAIRealtime(AzureChatCompletion):
query_params: RealtimeQueryParams | None = None,
user_api_key_dict: object | None = None,
litellm_metadata: dict | None = None,
realtime_mode: str = "realtime",
bridges_beta_client_to_ga: bool = False,
drop_params: bool = False,
):
import websockets
from websockets.asyncio.client import ClientConnection
if api_base is None:
raise ValueError("api_base is required for Azure OpenAI calls")
backend_uses_beta_protocol: Final = realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1")
backend_uses_beta_protocol: Final = not _is_ga_protocol(realtime_protocol)
if api_version is None and backend_uses_beta_protocol:
raise ValueError("api_version is required for Azure OpenAI calls")
@ -181,6 +231,7 @@ class AzureOpenAIRealtime(AzureChatCompletion):
api_version,
realtime_protocol=realtime_protocol,
query_params=query_params,
realtime_mode=realtime_mode,
)
auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token)
@ -204,6 +255,9 @@ class AzureOpenAIRealtime(AzureChatCompletion):
force_transcription_model=(
model if (query_params or {}).get("intent") == "transcription" else None
),
translation_session=realtime_mode == "translation",
bridges_beta_client_to_ga=bridges_beta_client_to_ga,
drop_params=drop_params,
)
await realtime_streaming.bidirectional_forward()

38
litellm/llms/azure/realtime/http_transformation.py
View file

@ -1,11 +1,20 @@
"""Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls)."""
from collections.abc import Mapping
from typing import Final
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.azure.common_utils import get_azure_ad_token
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
_CALLER_AUTH_HEADERS: Final = frozenset({"authorization", "api-key"})
def _azure_resource_base(api_base: str) -> str:
return api_base.rstrip("/").removesuffix("/openai/v1").removesuffix("/openai")
def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str:
@ -21,6 +30,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
def get_api_key(self, api_key: str | None, **kwargs: object) -> str:
return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
def get_extra_headers(
self,
litellm_params: GenericLiteLLMParams,
api_key: str,
extra_headers: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
if api_key or any(name.lower() in _CALLER_AUTH_HEADERS for name in extra_headers or {}):
return extra_headers
azure_ad_token: Final = get_azure_ad_token(litellm_params)
if azure_ad_token is None:
return extra_headers
return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"}
def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets")
@ -30,20 +52,30 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
model: str,
api_key: str | None = None,
) -> dict:
return {
validated_headers: Final = {
**headers,
"api-key": api_key or "",
"Content-Type": "application/json",
}
if api_key:
validated_headers["api-key"] = api_key
return validated_headers
def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base: Final = self.get_api_base(api_base)
base: Final = _azure_resource_base(self.get_api_base(api_base))
version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}"
def get_translation_client_secret_url(
self, api_base: str | None, model: str, api_version: str | None = None
) -> str:
return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/client_secrets")
def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/calls")
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
return {
"Authorization": f"Bearer {ephemeral_key}",

20
litellm/llms/azure_ai/chat/transformation.py
View file

@ -10,7 +10,11 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
declared_prompt_cache_breakpoint,
routed_model_info,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
content_parts,
convert_content_list_to_str,
@ -125,18 +129,14 @@ def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool)
def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool:
model_info: Final = routed_model_info(litellm_params.get("model_info"))
declared: Final = declared_prompt_cache_breakpoint(model_info)
if declared is not None:
return declared
base_model: Final = litellm_params.get("base_model")
named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,)
model_info: Final = litellm_params.get("model_info")
if not isinstance(model_info, Mapping):
return any(
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models
)
deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint")
if isinstance(deployment_flag, bool):
return deployment_flag
return any(
AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai")
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai", model_info=model_info)
for name in named_models
)

119
litellm/llms/base_llm/audio_transcription/transformation.py
View file

@ -1,9 +1,30 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Any,
Final,
cast, # noqa: TID251 # SDK file handles have no runtime validation schema
)
import httpx
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
from openai.types.audio import (
Transcription,
TranscriptionDiarized,
TranscriptionStreamEvent,
TranscriptionVerbose,
)
from openai.types.audio.transcription_create_params import (
TranscriptionCreateParamsBase,
TranscriptionCreateParamsStreaming,
)
from pydantic import BaseModel, TypeAdapter
from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.base_llm.chat.transformation import BaseConfig
from litellm.types.llms.openai import (
AllMessageValues,
@ -11,6 +32,31 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
_TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object])
_TRANSCRIPTION_TEXT: Final = TypeAdapter(str)
_TRANSCRIPTION_BODY: Final = TypeAdapter(dict[str, object])
_TRANSCRIPTION_TYPES_BY_FORMAT: Final = MappingProxyType(
{"verbose_json": TranscriptionVerbose, "diarized_json": TranscriptionDiarized}
)
def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]:
request_data: Final = _TRANSCRIPTION_REQUEST.validate_python(data)
extension_keys: Final = ("keywords", "languages")
extension_body: Final = {key: request_data[key] for key in extension_keys if key in request_data}
if not extension_body:
return request_data
existing_extra_body: Final = request_data.get("extra_body")
return {
**{key: value for key, value in request_data.items() if key not in extension_keys},
"extra_body": {
**(as_str_mapping(existing_extra_body) or {}),
**extension_body,
},
}
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
@ -20,6 +66,77 @@ else:
LiteLLMLoggingObj = Any
def sdk_compatible_transcription_response(response: object) -> BaseModel | str:
if isinstance(response, BaseModel):
return response
return _TRANSCRIPTION_TEXT.validate_python(response, strict=True)
def _content_type(http_response: httpx.Response) -> str:
return next(iter(http_response.headers.get_list("content-type")), "")
def _is_event_stream(http_response: httpx.Response) -> bool:
return _content_type(http_response).startswith("text/event-stream")
def _unstreamed_transcription_response(http_response: httpx.Response, response_format: object) -> BaseModel | str:
if not _content_type(http_response).startswith("application/json"):
return http_response.text
transcription_type: Final = _TRANSCRIPTION_TYPES_BY_FORMAT.get(str(response_format), Transcription)
return transcription_type.model_construct(None, **_TRANSCRIPTION_BODY.validate_json(http_response.content))
def _non_streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsBase:
return cast( # cast-ok: SDK file handles have no runtime validation schema
TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"}
)
def _streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsStreaming:
return cast( # cast-ok: SDK file handles have no runtime validation schema
TranscriptionCreateParamsStreaming, data
)
def create_sdk_transcription(
client: OpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout
) -> tuple[dict[str, str], Stream[TranscriptionStreamEvent] | BaseModel | str]:
if not transcription_stream_requested(data.get("stream")):
raw_response: Final = client.audio.transcriptions.with_raw_response.create(
**_non_streaming_params(data), timeout=timeout
)
return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse())
raw_stream: Final = client.audio.transcriptions.with_raw_response.create(
**_streaming_params({**data, "stream": True}), timeout=timeout
)
if _is_event_stream(raw_stream.http_response):
return dict(raw_stream.headers), raw_stream.parse(to=Stream[TranscriptionStreamEvent])
raw_stream.http_response.read()
return dict(raw_stream.headers), _unstreamed_transcription_response(
raw_stream.http_response, data.get("response_format")
)
async def acreate_sdk_transcription(
client: AsyncOpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout
) -> tuple[dict[str, str], AsyncStream[TranscriptionStreamEvent] | BaseModel | str]:
if not transcription_stream_requested(data.get("stream")):
raw_response: Final = await client.audio.transcriptions.with_raw_response.create(
**_non_streaming_params(data), timeout=timeout
)
return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse())
raw_stream: Final = await client.audio.transcriptions.with_raw_response.create(
**_streaming_params({**data, "stream": True}), timeout=timeout
)
if _is_event_stream(raw_stream.http_response):
return dict(raw_stream.headers), raw_stream.parse(to=AsyncStream[TranscriptionStreamEvent])
await raw_stream.http_response.aread()
return dict(raw_stream.headers), _unstreamed_transcription_response(
raw_stream.http_response, data.get("response_format")
)
@dataclass
class AudioTranscriptionRequestData:
"""

21
litellm/llms/base_llm/realtime/http_transformation.py
View file

@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow:
"""
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Final
import httpx
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.types.router import GenericLiteLLMParams
class BaseRealtimeHTTPConfig(ABC):
@ -50,6 +53,14 @@ class BaseRealtimeHTTPConfig(ABC):
explicit api_key → litellm.api_key → env var → ""
"""
def get_extra_headers(
self,
litellm_params: GenericLiteLLMParams,
api_key: str,
extra_headers: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
return extra_headers
# ------------------------------------------------------------------ #
# client_secrets endpoint #
# ------------------------------------------------------------------ #
@ -64,6 +75,12 @@ class BaseRealtimeHTTPConfig(ABC):
base = base.removesuffix("/v1")
return join_api_base(base, "v1/realtime/transcription_sessions")
def get_translation_client_secret_url(
self, api_base: str | None, model: str, api_version: str | None = None
) -> str:
base: Final = (api_base or "").rstrip("/")
return f"{base}/v1/realtime/translations/client_secrets"
@abstractmethod
def validate_environment(
self,
@ -86,6 +103,10 @@ class BaseRealtimeHTTPConfig(ABC):
"""Return the full URL for POST /realtime/calls (SDP exchange)."""
return join_api_base(api_base or "", "v1/realtime/calls")
def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base: Final = (api_base or "").rstrip("/")
return f"{base}/v1/realtime/translations/calls"
def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
"""
Build headers for the realtime_calls POST.

197
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -23,6 +23,11 @@ from litellm.constants import (
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.integrations.anthropic_cache_control_hook import (
MAX_CACHE_CONTROL_BLOCKS,
AnthropicCacheControlHook,
configured_injection_points,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
@ -131,6 +136,7 @@ from ..common_utils import (
bedrock_capability_model,
bedrock_converse_supports_parallel_tool_use_config,
bedrock_model_accepts_cache_points,
bedrock_model_known_to_accept_cache_points,
bedrock_reasoning_effort_disabled,
bedrock_rejects_stop_sequences,
get_anthropic_beta_from_headers,
@ -204,6 +210,11 @@ def _has_native_text_format(output_config: object) -> bool:
return False
def _cache_mark_kept(block: Mapping[str, object], keeps_cache_marks: bool) -> Mapping[str, object]:
cache_control: Final = block.get("cache_control")
return {"cache_control": cache_control} if keeps_cache_marks and cache_control is not None else {}
class AmazonConverseConfig(BaseConfig):
"""
Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@ -243,7 +254,7 @@ class AmazonConverseConfig(BaseConfig):
@staticmethod
def _convert_consecutive_user_messages_to_guarded_text(
messages: list[AllMessageValues], optional_params: dict
messages: list[AllMessageValues], optional_params: dict, keeps_cache_marks: bool = False
) -> list[AllMessageValues]:
"""
Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present
@ -281,7 +292,11 @@ class AmazonConverseConfig(BaseConfig):
new_content = []
for item in content:
if isinstance(item, dict) and item.get("type") == "text":
new_item = {"type": "guarded_text", "text": item["text"]}
new_item = {
"type": "guarded_text",
"text": item["text"],
**_cache_mark_kept(_JSON_OBJECT.validate_python(item), keeps_cache_marks),
}
new_content.append(new_item)
else:
new_content.append(item)
@ -289,7 +304,13 @@ class AmazonConverseConfig(BaseConfig):
messages_copy[user_message_index]["content"] = new_content
elif isinstance(content, str):
# If content is a string, convert it to guarded_text
messages_copy[user_message_index]["content"] = [{"type": "guarded_text", "text": content}]
messages_copy[user_message_index]["content"] = [
{
"type": "guarded_text",
"text": content,
**_cache_mark_kept(_JSON_OBJECT.validate_python(user_message), keeps_cache_marks),
}
]
return messages_copy
@ -1420,6 +1441,7 @@ class AmazonConverseConfig(BaseConfig):
block_type: Literal["system"],
model: str | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> SystemContentBlock | None:
pass
@ -1433,6 +1455,7 @@ class AmazonConverseConfig(BaseConfig):
block_type: Literal["content_block"],
model: str | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> ContentBlock | None:
pass
@ -1445,9 +1468,10 @@ class AmazonConverseConfig(BaseConfig):
block_type: Literal["system", "content_block"],
model: str | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> SystemContentBlock | ContentBlock | None:
cache_control: Final = message_block.get("cache_control", None)
if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model):
if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model, model_info):
return None
cache_point: Final = self._build_cache_point_block(cache_control, model, deployment_model)
@ -1592,8 +1616,80 @@ class AmazonConverseConfig(BaseConfig):
return ChatCompletionTextObject(type="text", text=text)
return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control)
def _messages_with_cache_marks(
self,
messages: Sequence[AllMessageValues],
model: str,
deployment_model: str,
drop_params: bool,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> Sequence[AllMessageValues]:
if not bedrock_model_known_to_accept_cache_points(model, deployment_model, model_info):
return messages
markable: Final = (
AnthropicCacheControlHook.prompt_cache_breakpoints_dropped_from_tool_results(messages, drop_params, model)
if "nova" in model.lower()
else messages
)
return AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
markable,
drop_params,
AnthropicCacheControlHook.count_converse_cache_points,
lambda: self._tool_cache_points_on_the_wire(
model, deployment_model, optional_params, litellm_params, model_info
),
)
def _tool_cache_points_on_the_wire(
self,
model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> int:
tools: Final = optional_params.get("tools")
bedrock_tools, _ = self._process_tools_and_beta(
tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model, model_info
)
injects_tool_config_point: Final = bool(bedrock_tools) and any(
point.get("location") == "tool_config"
for point in configured_injection_points(optional_params.get("cache_control_injection_points"))
)
return sum(1 for tool in bedrock_tools if "cachePoint" in tool) + int(injects_tool_config_point)
def _guarded_text_keeps_cache_marks(
self,
messages: Sequence[AllMessageValues],
system_content_blocks: Sequence[SystemContentBlock],
model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> bool:
if "guardrailConfig" not in optional_params or not bedrock_model_known_to_accept_cache_points(
model, deployment_model, model_info
):
return False
message_cache_points: Final = sum(map(AnthropicCacheControlHook.count_converse_cache_points, messages))
if message_cache_points == 0:
return False
cache_points: Final = (
sum(1 for block in system_content_blocks if "cachePoint" in block)
+ message_cache_points
+ self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params, model_info)
)
return cache_points <= MAX_CACHE_CONTROL_BLOCKS
def _transform_system_message(
self, messages: list[AllMessageValues], model: str | None = None, deployment_model: str | None = None
self,
messages: list[AllMessageValues],
model: str | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> tuple[list[AllMessageValues], list[SystemContentBlock]]:
leading_count: Final = next(
(i for i, m in enumerate(messages) if not is_system_message(m)),
@ -1609,7 +1705,7 @@ class AmazonConverseConfig(BaseConfig):
if isinstance(content, str) and content:
system_content_blocks.append(SystemContentBlock(text=content))
cache_block = self.get_cache_point_block(
message, block_type="system", model=model, deployment_model=deployment_model
message, block_type="system", model=model, deployment_model=deployment_model, model_info=model_info
)
if cache_block:
system_content_blocks.append(cache_block)
@ -1618,7 +1714,11 @@ class AmazonConverseConfig(BaseConfig):
if m.get("type") == "text" and m.get("text"):
system_content_blocks.append(SystemContentBlock(text=m["text"]))
cache_block = self.get_cache_point_block(
m, block_type="system", model=model, deployment_model=deployment_model
m,
block_type="system",
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
if cache_block:
system_content_blocks.append(cache_block)
@ -1816,6 +1916,7 @@ class AmazonConverseConfig(BaseConfig):
additional_request_params: dict,
litellm_params: Mapping[str, object] | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> tuple[list[ToolBlock], list]:
"""Process tools and collect anthropic_beta values."""
bedrock_tools: list[ToolBlock] = []
@ -1877,6 +1978,7 @@ class AmazonConverseConfig(BaseConfig):
_tools_the_model_accepts(regular_tools, model, litellm_params),
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
# Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
@ -1946,6 +2048,7 @@ class AmazonConverseConfig(BaseConfig):
_tools_the_model_accepts(_TOOLS_AS_SENT.validate_python(filtered_tools), model, litellm_params),
model=model,
deployment_model=deployment_model,
model_info=model_info,
)
# Append pre-formatted tools (systemTool etc.) after transformation
@ -2021,6 +2124,7 @@ class AmazonConverseConfig(BaseConfig):
drop_params: bool = False,
litellm_params: Mapping[str, object] | None = None,
deployment_model: str | None = None,
model_info: Mapping[str, object] | None = None,
) -> CommonRequestObject:
# Drop thinking param if thinking is enabled but thinking_blocks are missing
# This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
@ -2059,7 +2163,7 @@ class AmazonConverseConfig(BaseConfig):
# Process tools and collect beta values
bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
original_tools, model, headers, additional_request_params, litellm_params, deployment_model
original_tools, model, headers, additional_request_params, litellm_params, deployment_model, model_info
)
requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict
body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict
@ -2077,7 +2181,7 @@ class AmazonConverseConfig(BaseConfig):
if (
cache_injection_points
and len(bedrock_tools) > 0
and bedrock_model_accepts_cache_points(model, deployment_model)
and bedrock_model_accepts_cache_points(model, deployment_model, model_info)
):
for point in cache_injection_points:
if point.get("location") == "tool_config":
@ -2137,22 +2241,27 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
self,
messages: Sequence[AllMessageValues],
capability_model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
messages_within_reserved_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
return messages_within_reserved_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)),
self._tool_cache_points_on_the_wire(
capability_model, deployment_model, optional_params, litellm_params, model_info
),
)
async def _async_transform_request(
@ -2163,27 +2272,53 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict[str, object],
headers: dict | None = None,
) -> RequestObject:
from litellm.integrations.anthropic_cache_control_hook import routed_model_info
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
marked_messages: Final = self._messages_with_cache_marks(
messages,
capability_model,
model,
drops_unsupported_params(litellm_params.get("drop_params")),
json_optional_params,
litellm_params,
model_info,
)
messages, system_content_blocks = self._transform_system_message(
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
)
),
model=capability_model,
deployment_model=model,
model_info=model_info,
)
# Convert last user message to guarded_text if guardrailConfig is present
guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params)
messages = self._convert_consecutive_user_messages_to_guarded_text(
self._handle_orphaned_tool_blocks(messages, optional_params), optional_params
guard_input,
optional_params,
keeps_cache_marks=self._guarded_text_keeps_cache_marks(
guard_input,
system_content_blocks,
capability_model,
model,
json_optional_params,
litellm_params,
model_info,
),
)
## TRANSFORMATION ##
_data: Final[CommonRequestObject] = self._transform_request_helper(
model=capability_model,
deployment_model=model,
model_info=model_info,
system_content_blocks=system_content_blocks,
optional_params=optional_params,
messages=messages,
@ -2198,6 +2333,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
model=capability_model,
deployment_model=model,
model_info=model_info,
llm_provider="bedrock_converse",
user_continue_message=litellm_params.pop("user_continue_message", None),
base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)),
@ -2250,26 +2386,52 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict[str, object],
headers: dict | None = None,
) -> RequestObject:
from litellm.integrations.anthropic_cache_control_hook import routed_model_info
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
marked_messages: Final = self._messages_with_cache_marks(
messages,
capability_model,
model,
drops_unsupported_params(litellm_params.get("drop_params")),
json_optional_params,
litellm_params,
model_info,
)
messages, system_content_blocks = self._transform_system_message(
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
)
),
model=capability_model,
deployment_model=model,
model_info=model_info,
)
# Convert last user message to guarded_text if guardrailConfig is present
guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params)
messages = self._convert_consecutive_user_messages_to_guarded_text(
self._handle_orphaned_tool_blocks(messages, optional_params), optional_params
guard_input,
optional_params,
keeps_cache_marks=self._guarded_text_keeps_cache_marks(
guard_input,
system_content_blocks,
capability_model,
model,
json_optional_params,
litellm_params,
model_info,
),
)
_data: Final[CommonRequestObject] = self._transform_request_helper(
model=capability_model,
deployment_model=model,
model_info=model_info,
system_content_blocks=system_content_blocks,
optional_params=optional_params,
messages=messages,
@ -2285,6 +2447,7 @@ class AmazonConverseConfig(BaseConfig):
messages=messages,
model=capability_model,
deployment_model=model,
model_info=model_info,
llm_provider="bedrock_converse",
user_continue_message=litellm_params.pop("user_continue_message", None),
base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)),

5
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py
View file

@ -13,8 +13,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_image_obj,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
RemoteMedia,
async_inline_remote_media,
convert_url_to_base64,
inline_every_remote_url,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import (
@ -117,6 +119,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
def should_strip_billing_metadata(self) -> bool:
return True
def inlines_remote_media(self, media: RemoteMedia) -> bool:
return inline_every_remote_url(media)
def get_supported_openai_params(self, model: str) -> list[str]:
return AnthropicConfig.get_supported_openai_params(self, model)

92
litellm/llms/bedrock/common_utils.py
View file

@ -33,6 +33,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.core_helpers import lax_bool
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
from litellm.litellm_core_utils.url_utils import ensure_api_route
@ -1515,7 +1516,9 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
)
def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str | None = None) -> bool:
def bedrock_model_accepts_cache_points(
model: str | None, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool:
"""
Whether Converse ``cachePoint`` blocks may be sent to this model.
@ -1535,43 +1538,102 @@ def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str
never silently degrade. ``litellm.utils.supports_prompt_caching`` is not reusable here:
it returns False for unmapped models, the opposite polarity. ``deployment_model`` is the
model as configured when ``model`` is its ``base_model``: its own declaration wins.
A routed call passes its deployment's ``model_info``, whose breakpoint flag wins outright and
whose ``supports_prompt_caching`` wins over the catalog's for a known model. Other flags come
from the catalog as loaded, since the registered keys are shared by every deployment of the
same model string and carry whichever sibling registered last.
"""
if model is None:
return True
deployment_declared: Final = (
None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model)
None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model, model_info)
)
declared: Final = (
deployment_declared if deployment_declared is not None else bedrock_declared_cache_point_support(model)
deployment_declared
if deployment_declared is not None
else bedrock_declared_cache_point_support(model, model_info)
)
return declared is not False
def bedrock_declared_cache_point_support(model: str) -> bool | None:
def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None:
if _OPENAI_FAMILY_MODEL_RE.search(model):
return False
map_keys: Final = (model, get_bedrock_base_model(model))
explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
if explicit_marker_support is not None:
return explicit_marker_support
return _mapped_prompt_caching_support(model, model_info)
def bedrock_model_known_to_accept_cache_points(
model: str, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool:
"""Like ``bedrock_model_accepts_cache_points``, but only for the Claude and Nova families Bedrock documents
as taking cache checkpoints, unless the entry sets ``supports_prompt_cache_breakpoint``: an OpenAI
``prompt_cache_breakpoint`` reaches every model of a mixed fleet, not only the ones a client marked for
Bedrock caching, and other families price implicit cache reads yet may reject the cachePoint field."""
deployment_declared: Final = (
None if deployment_model is None else _declared_marker_support(deployment_model, model_info)
)
declared: Final = (
deployment_declared if deployment_declared is not None else _declared_marker_support(model, model_info)
)
return declared is True
def _declared_marker_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
if _OPENAI_FAMILY_MODEL_RE.search(model):
return False
explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
if explicit_marker_support is not None:
return explicit_marker_support
mapped: Final = _mapped_prompt_caching_support(model, model_info)
if not _documents_cache_checkpoints(model):
return None if mapped is None else False
return mapped is not False
def _documents_cache_checkpoints(model: str) -> bool:
return _is_claude(model) or get_bedrock_base_model(model).startswith("amazon.nova")
def _explicit_cache_point_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
if deployment_flag is not None:
return deployment_flag is True
catalog_entry: Final = _catalog_entry_reader(model_info)
registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/"))
explicit_marker_support: Final = next(
return next(
(
entry.get("supports_prompt_cache_breakpoint") is True
for key in (*registered_keys, *map_keys)
if (entry := litellm.model_cost.get(key)) is not None
and entry.get("supports_prompt_cache_breakpoint") is not None
for key in (*registered_keys, model, get_bedrock_base_model(model))
if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None
),
None,
)
if explicit_marker_support is not None:
return explicit_marker_support
if not any(key in litellm.model_cost for key in map_keys):
def _mapped_prompt_caching_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
catalog_entry: Final = _catalog_entry_reader(model_info)
map_keys: Final = (model, get_bedrock_base_model(model))
if not any(catalog_entry(key) is not None for key in map_keys):
return None
deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching"))
if deployment_caching is not None:
return deployment_caching
return any(
entry.get("supports_prompt_caching") is True
for key in map_keys
if (entry := litellm.model_cost.get(key)) is not None
entry.get("supports_prompt_caching") is True for key in map_keys if (entry := catalog_entry(key)) is not None
)
def _catalog_entry_reader(model_info: Mapping[str, object] | None) -> Callable[[str], Mapping[str, object] | None]:
from litellm.utils import catalog_with_runtime_registrations
if model_info is not None:
return catalog_with_runtime_registrations().get
return _model_cost_entry
def bedrock_supports_tool_search(model: str) -> bool:
"""
Whether Bedrock InvokeModel admits the ``tool_search_tool_*`` tool types on ``model``.

164
litellm/llms/bedrock/realtime/handler.py
View file

@ -9,7 +9,7 @@ import contextlib
import importlib.metadata
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, NoReturn, Protocol, runtime_checkable
@ -34,12 +34,15 @@ from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateR
from litellm.types.llms.bedrock import AwsAuthParams
from litellm.types.llms.openai import OpenAIRealtimeEvents
from litellm.types.realtime import RealtimeResponseTransformInput
from litellm.types.utils import StandardLoggingPayloadErrorInformation
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, requested_modalities
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_REQUEST_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_SPEND_LOG_METADATA_KEYS: Final = ("litellm_metadata", "metadata")
_EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
@ -61,6 +64,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType(
)
def _stream_ended_before_session_closed() -> BedrockError:
return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed")
def _as_bedrock_error(error: BaseException) -> BaseException:
status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__)
if status_code is None:
@ -165,6 +172,12 @@ class _BridgeOutcome:
client_disconnected: bool
@dataclass(frozen=True, slots=True)
class SessionEndSignals:
input_closing: asyncio.Event = field(default_factory=asyncio.Event)
output_failed: asyncio.Event = field(default_factory=asyncio.Event)
async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]:
if initial_message is not None:
yield initial_message
@ -188,8 +201,8 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
raise BedrockError(
status_code=400,
message=(
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
(f"{committed_failure}. " if committed_failure else "")
+ "Bedrock realtime session already committed to a provider stream; it cannot be replayed"
),
)
@ -201,11 +214,48 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
return pending if isinstance(pending, str) else None
def _spend_log_metadata(litellm_params: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None:
return next(
(
(key, metadata)
for key in _SPEND_LOG_METADATA_KEYS
if (metadata := _REQUEST_MAPPING_ADAPTER.validate_python(litellm_params.get(key)))
),
None,
)
def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None:
"""The spend row of a session the provider ended reads failure while it still bills the forwarded events."""
litellm_params: Final = _REQUEST_MAPPING_ADAPTER.validate_python(
logging_obj.model_call_details.get("litellm_params")
)
if litellm_params is None:
return
spend_log_metadata: Final = _spend_log_metadata(litellm_params)
if spend_log_metadata is None:
return
status_code: Final[object] = getattr(error, "status_code", None)
error_information: Final[StandardLoggingPayloadErrorInformation] = {
"error_code": "" if status_code is None else str(status_code),
"error_class": type(error).__name__,
"error_message": redact_string(str(error)),
"llm_provider": "bedrock",
}
metadata_key, metadata = spend_log_metadata
logging_obj.model_call_details["litellm_params"] = { # rebind-ok: the spend row reads the logging object
**litellm_params,
metadata_key: {**metadata, "status": "failure", "error_information": error_information},
}
def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
error: Final = _as_bedrock_error(failure)
verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
if _is_session_committed(scope):
scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error))
if error is failure:
raise failure
raise error from failure
@ -267,6 +317,25 @@ async def _reject_session_update(
return True
async def _close_bedrock_input(
bedrock_stream: BedrockBidirectionalStream,
close_messages: Sequence[str],
send_to_bedrock: Callable[[str], Awaitable[None]],
timeout: float,
) -> None:
async def flush_and_close() -> None:
for close_message in close_messages:
with contextlib.suppress(Exception):
await send_to_bedrock(close_message)
with contextlib.suppress(Exception):
await bedrock_stream.input_stream.close()
try:
await asyncio.wait_for(flush_and_close(), timeout)
except (asyncio.TimeoutError, TimeoutError):
verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout)
class BedrockRealtime(BaseAWSLLM):
"""Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
@ -278,6 +347,18 @@ class BedrockRealtime(BaseAWSLLM):
super().__init__()
self._sdk_version_lookup: Final = sdk_version_lookup
self._close_timeout: Final = close_timeout
self._failed_input_closes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs to running closes
def _close_failed_input_in_background(
self,
bedrock_stream: BedrockBidirectionalStream,
send_to_bedrock: Callable[[str], Awaitable[None]],
) -> None:
close_task: Final = asyncio.create_task(
_close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout)
)
self._failed_input_closes.add(close_task)
close_task.add_done_callback(self._failed_input_closes.discard)
async def async_realtime(
self,
@ -433,23 +514,26 @@ class BedrockRealtime(BaseAWSLLM):
if _should_log_event(leftover_event)
),
)
if outcome.provider_failure is not None and outcome.client_disconnected:
verbose_proxy_logger.debug(
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
)
provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure
if logged_events:
if provider_failure is not None:
_record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure))
usage_logging_obj: Final = logging_obj if provider_failure is None else logging_obj.attempt_scoped_copy()
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
logging_obj.dispatch_success_handlers(
usage_logging_obj.dispatch_success_handlers(
list(logged_events),
prefer_async_handlers=True,
)
)
logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True
if outcome.provider_failure is None:
if provider_failure is None:
return
if outcome.client_disconnected:
verbose_proxy_logger.debug(
"Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure
)
return
_raise_provider_failure(websocket.scope, outcome.provider_failure)
_raise_provider_failure(websocket.scope, provider_failure)
async def _bridge(
self,
@ -463,16 +547,34 @@ class BedrockRealtime(BaseAWSLLM):
) -> _BridgeOutcome:
"""Run both forwarding directions until the client leaves or either side fails."""
logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend
signals: Final = SessionEndSignals()
async def collect_logged_events() -> None:
async for event in self._forward_bedrock_to_client(
bedrock_stream, websocket, transformation_config, model, logging_obj, session_state
):
logged.append(event)
try:
async for event in self._forward_bedrock_to_client(
bedrock_stream,
websocket,
transformation_config,
model,
logging_obj,
session_state,
signals=signals,
):
logged.append(event)
except Exception:
signals.output_failed.set()
raise
client_task: Final = asyncio.create_task(
self._forward_client_to_bedrock(
websocket, bedrock_stream, transformation_config, model, session_state, logging_obj, initial_message
websocket,
bedrock_stream,
transformation_config,
model,
session_state,
logging_obj,
initial_message,
signals=signals,
)
)
bedrock_task: Final = asyncio.create_task(collect_logged_events())
@ -506,6 +608,8 @@ class BedrockRealtime(BaseAWSLLM):
session_state: RealtimeResponseTransformInput,
logging_obj: LiteLLMLogging | None = None,
initial_message: str | None = None,
*,
signals: SessionEndSignals,
) -> None:
"""Forward messages from client WebSocket to Bedrock stream.
@ -521,7 +625,10 @@ class BedrockRealtime(BaseAWSLLM):
async def send_to_bedrock(bedrock_message: str) -> None:
event: Final = build_input_chunk(bedrock_message.encode("utf-8"))
await bedrock_stream.input_stream.send(event)
try:
await bedrock_stream.input_stream.send(event)
except Exception as e:
raise _stream_ended_before_session_closed() from e
verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200])
try:
@ -553,20 +660,13 @@ class BedrockRealtime(BaseAWSLLM):
client_ws, transformation_config, model, logging_obj, parsed_client_message
):
break
signals.input_closing.set()
finally:
async def close_bedrock_input() -> None:
for close_message in transformation_config.session_close_messages():
with contextlib.suppress(Exception):
await send_to_bedrock(close_message)
with contextlib.suppress(Exception):
await bedrock_stream.input_stream.close()
try:
await asyncio.wait_for(close_bedrock_input(), self._close_timeout)
except (asyncio.TimeoutError, TimeoutError):
verbose_proxy_logger.debug(
"Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout
if signals.output_failed.is_set():
self._close_failed_input_in_background(bedrock_stream, send_to_bedrock)
else:
await _close_bedrock_input(
bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout
)
async def _forward_bedrock_to_client(
@ -577,6 +677,8 @@ class BedrockRealtime(BaseAWSLLM):
model: str,
logging_obj: LiteLLMLogging,
session_state: RealtimeResponseTransformInput,
*,
signals: SessionEndSignals,
) -> AsyncIterator[OpenAIRealtimeEvents]:
"""Forward messages from Bedrock to the client, yielding the ones to record for spend logging.
@ -597,6 +699,8 @@ class BedrockRealtime(BaseAWSLLM):
result = await output[1].receive()
if result is None:
if not signals.input_closing.is_set():
raise _stream_ended_before_session_closed()
verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended")
with contextlib.suppress(Exception):
await client_ws.close()

109
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -7097,7 +7097,7 @@ class BaseLLMHTTPHandler:
timeout: float | httpx.Timeout,
provider_config: BaseRealtimeHTTPConfig | None = None,
model: str | None = None,
extra_headers: dict[str, object] | None = None,
extra_headers: Mapping[str, object] | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
api_version: str | None = None,
) -> httpx.Response:
@ -7130,7 +7130,7 @@ class BaseLLMHTTPHandler:
timeout: float | httpx.Timeout,
provider_config: BaseRealtimeHTTPConfig | None = None,
model: str | None = None,
extra_headers: dict[str, object] | None = None,
extra_headers: Mapping[str, object] | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
api_version: str | None = None,
) -> httpx.Response:
@ -7149,9 +7149,8 @@ class BaseLLMHTTPHandler:
api_version=api_version,
)
async def _async_realtime_session_post(
async def async_realtime_translation_client_secret_handler(
self,
endpoint: Literal["client_secrets", "transcription_sessions"],
api_base: str,
api_key: str,
request_data: dict[str, object],
@ -7159,7 +7158,35 @@ class BaseLLMHTTPHandler:
timeout: float | httpx.Timeout,
provider_config: BaseRealtimeHTTPConfig | None = None,
model: str | None = None,
extra_headers: dict[str, object] | None = None,
extra_headers: Mapping[str, object] | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
api_version: str | None = None,
) -> httpx.Response:
return await self._async_realtime_session_post(
endpoint="translation_client_secrets",
api_base=api_base,
api_key=api_key,
request_data=request_data,
logging_obj=logging_obj,
timeout=timeout,
provider_config=provider_config,
model=model,
extra_headers=extra_headers,
client=client,
api_version=api_version,
)
async def _async_realtime_session_post(
self,
endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"],
api_base: str,
api_key: str,
request_data: dict[str, object],
logging_obj: LiteLLMLoggingObj,
timeout: float | httpx.Timeout,
provider_config: BaseRealtimeHTTPConfig | None = None,
model: str | None = None,
extra_headers: Mapping[str, object] | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
api_version: str | None = None,
) -> httpx.Response:
@ -7182,13 +7209,20 @@ class BaseLLMHTTPHandler:
url = provider_config.get_transcription_session_url(
api_base=api_base, model=model or "", api_version=api_version
)
elif endpoint == "translation_client_secrets":
url = provider_config.get_translation_client_secret_url(
api_base=api_base, model=model or "", api_version=api_version
)
else:
url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version)
headers: dict[str, object] = provider_config.validate_environment(
headers={}, model=model or "", api_key=api_key
)
else:
url = join_api_base(api_base, f"v1/realtime/{endpoint}")
endpoint_path: Final = (
"translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint
)
url = join_api_base(api_base, f"v1/realtime/{endpoint_path}")
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
@ -7222,6 +7256,12 @@ class BaseLLMHTTPHandler:
)
raise
@staticmethod
def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler:
if isinstance(client, AsyncHTTPHandler):
return client
return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI)
async def async_realtime_calls_handler(
self,
api_base: str,
@ -7231,10 +7271,11 @@ class BaseLLMHTTPHandler:
timeout: float | httpx.Timeout,
provider_config: BaseRealtimeHTTPConfig | None = None,
model: str | None = None,
session_config: dict[str, object] | None = None,
extra_headers: dict[str, object] | None = None,
session_config: Mapping[str, object] | None = None,
extra_headers: Mapping[str, object] | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
api_version: str | None = None,
translation: bool = False,
) -> httpx.Response:
"""
Forward POST /v1/realtime/calls (SDP exchange) to upstream provider.
@ -7246,18 +7287,30 @@ class BaseLLMHTTPHandler:
- sdp: the SDP offer (text)
- session: JSON string with {"type": "realtime", "model": "...", ...}
"""
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders.OPENAI,
)
else:
async_httpx_client = client
session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally
**(session_config or {})
}
if "type" not in session_data:
session_data["type"] = "translation" if translation else "realtime"
if "model" not in session_data and model:
session_data["model"] = model
sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body
async_httpx_client: Final = self._get_realtime_async_http_client(client)
if provider_config is not None:
url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version)
url = (
provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version)
if translation
else provider_config.get_realtime_calls_url(
api_base=api_base, model=model or "", api_version=api_version
)
)
headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key)
else:
url = join_api_base(api_base, "v1/realtime/calls")
path: Final = "translations/calls" if translation else "calls"
url = join_api_base(api_base, f"v1/realtime/{path}")
headers = {
"Authorization": f"Bearer {openai_ephemeral_key}",
}
@ -7265,14 +7318,8 @@ class BaseLLMHTTPHandler:
if extra_headers:
headers.update(extra_headers)
# Build multipart form data: sdp + session JSON
session_data: Final = session_config or {}
if "type" not in session_data:
session_data["type"] = "realtime"
if "model" not in session_data and model:
session_data["model"] = model
sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body
if translation:
headers["Content-Type"] = "application/sdp"
files: Final = {
"sdp": (None, sdp_text, "text/plain"),
@ -7290,12 +7337,14 @@ class BaseLLMHTTPHandler:
)
try:
return await async_httpx_client.post(
url=url,
headers=headers,
files=files,
timeout=timeout,
)
if translation:
return await async_httpx_client.post(
url=url,
headers=headers,
content=sdp_text,
timeout=timeout,
)
return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout)
except Exception as e:
if provider_config is not None:
raise self._handle_error(

19
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -556,18 +556,23 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
"""
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
api.openai.com, or one opted in through ``prompt_cache_options`` or the cost
map) gets each Anthropic ``cache_control`` mark rewritten in place, and under
drop_params each content part's ``prompt_cache_breakpoint`` cut down to what chat
completions accepts. Any other endpoint keeps or strips the marks as before, and
under drop_params also loses every ``prompt_cache_breakpoint`` it cannot read.
api.openai.com, or one opted in through ``prompt_cache_options``, its own
``model_info``, or the cost map) gets each
Anthropic ``cache_control`` mark rewritten in place, and under drop_params each
content part's ``prompt_cache_breakpoint`` cut down to what chat completions
accepts. Any other endpoint keeps or strips the marks as before, and under
drop_params also loses every ``prompt_cache_breakpoint`` it cannot read.
"""
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, routed_model_info
custom_llm_provider: Final = _litellm_params_str(litellm_params, "custom_llm_provider")
api_base: Final = _litellm_params_str(litellm_params, "api_base")
reads_breakpoints: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, api_base, prompt_cache_options
model,
custom_llm_provider,
api_base,
prompt_cache_options,
routed_model_info(litellm_params.get("model_info")),
)
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
return (

20
litellm/llms/openai/live/handler.py
View file

@ -18,6 +18,7 @@ from litellm._logging import redact_secrets
from litellm.constants import (
OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS,
OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
REALTIME_SESSION_ENDED_ABNORMALLY_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
)
@ -316,12 +317,15 @@ async def _forward_live_backend_frames(
backend_websocket: LiveBackendWebSocket,
client_gone: asyncio.Event,
event_retention: _LiveEventRetention,
upstream_ended_abnormally: asyncio.Event,
) -> None:
from starlette.websockets import WebSocketDisconnect
from websockets.exceptions import ConnectionClosed
event_loop: Final = asyncio.get_running_loop()
async for message, close_deadline in _live_backend_messages(websocket, backend_websocket):
async for message, close_deadline in _live_backend_messages(
websocket, backend_websocket, upstream_ended_abnormally
):
if (event := _metering_event(message)) is not None:
event_retention.retain(event)
if client_gone.is_set():
@ -341,6 +345,7 @@ async def _forward_live_backend_frames(
async def _live_backend_messages(
websocket: LiveClientWebSocket,
backend_websocket: LiveBackendWebSocket,
upstream_ended_abnormally: asyncio.Event,
) -> AsyncIterator[tuple[str, float | None]]:
from starlette.websockets import WebSocketDisconnect
from websockets.exceptions import ConnectionClosed
@ -350,6 +355,8 @@ async def _live_backend_messages(
yield message, None
except ConnectionClosed as error:
upstream_close: Final = backend_close_from(error)
if not upstream_close.ended_cleanly:
upstream_ended_abnormally.set()
with suppress(WebSocketDisconnect, RuntimeError, ConnectionClosed, OSError):
await websocket.close(code=upstream_close.code, reason=upstream_close.reason)
return
@ -434,6 +441,7 @@ class OpenAILiveSessions(OpenAIRealtime):
"complete_input_dict": {"session_start": rewritten_start},
}
event_retention: Final = _LiveEventRetention()
session_ended_abnormally: Final = asyncio.Event()
try:
logging_obj.pre_call(input=None, api_key=api_key, additional_args=pre_call_args)
async with self._websocket_connector(
@ -448,7 +456,11 @@ class OpenAILiveSessions(OpenAIRealtime):
websocket=websocket,
backend_websocket=backend_websocket,
event_retention=event_retention,
upstream_ended_abnormally=session_ended_abnormally,
)
except Exception:
session_ended_abnormally.set()
raise
finally:
session_end_time: Final = self._clock()
retained_events: Final = event_retention.results()
@ -463,6 +475,10 @@ class OpenAILiveSessions(OpenAIRealtime):
logging_obj.model_call_details[ # rebind-ok: flags the shared logging object as already logged
REALTIME_SESSION_SUCCESS_LOGGED_KEY
] = True
if session_ended_abnormally.is_set():
logging_obj.model_call_details[ # rebind-ok: flags the shared logging object for the router
REALTIME_SESSION_ENDED_ABNORMALLY_KEY
] = True
except InvalidStatus as error:
await close_after_upstream_handshake_refusal(websocket, error.response.status_code)
except Exception as error: # noqa: BLE001 # any relay failure must still close the client socket with a redacted reason
@ -484,6 +500,7 @@ class OpenAILiveSessions(OpenAIRealtime):
websocket: LiveClientWebSocket,
backend_websocket: LiveBackendWebSocket,
event_retention: _LiveEventRetention,
upstream_ended_abnormally: asyncio.Event,
) -> None:
from websockets.exceptions import ConnectionClosed
@ -503,6 +520,7 @@ class OpenAILiveSessions(OpenAIRealtime):
backend_websocket=backend_websocket,
client_gone=client_gone,
event_retention=event_retention,
upstream_ended_abnormally=upstream_ended_abnormally,
)
)
client_gone_task: Final = asyncio.create_task(client_gone.wait())

15
litellm/llms/openai/realtime/handler.py
View file

@ -141,7 +141,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
return ssl_config
def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str:
def _construct_url(self, api_base: str, query_params: RealtimeQueryParams, realtime_mode: str = "realtime") -> str:
"""
Construct the backend websocket URL with the client's query parameters.
@ -158,7 +158,8 @@ class OpenAIRealtime(OpenAIChatCompletion):
api_base = api_base.replace("http://", "ws://")
url = URL(api_base)
# Set the correct path
url = url.copy_with(path="/v1/realtime")
path: Final = "/v1/realtime/translations" if realtime_mode == "translation" else "/v1/realtime"
url = url.copy_with(path=path)
upstream_params: Final = tuple(
(key, value) for key, value in query_params.items() if not (drops_model and key == "model")
)
@ -197,6 +198,8 @@ class OpenAIRealtime(OpenAIChatCompletion):
query_params: RealtimeQueryParams | None = None,
user_api_key_dict: object | None = None,
litellm_metadata: dict | None = None,
realtime_mode: str = "realtime",
drop_params: bool = False,
**kwargs: object,
):
import websockets
@ -210,13 +213,15 @@ class OpenAIRealtime(OpenAIChatCompletion):
# Use all query params if provided, else fallback to just model
if query_params is None:
query_params = {"model": model}
url: Final = self._construct_url(api_base, query_params)
url: Final = self._construct_url(api_base, query_params, realtime_mode=realtime_mode)
try:
# Get provider-specific SSL configuration
ssl_config: Final = self._get_ssl_config(url)
openai_beta_realtime: Final = client_sent_openai_beta_realtime_header(websocket)
openai_beta_realtime: Final = realtime_mode != "translation" and client_sent_openai_beta_realtime_header(
websocket
)
if not openai_beta_realtime:
verbose_logger.debug(
"OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). "
@ -253,6 +258,8 @@ class OpenAIRealtime(OpenAIChatCompletion):
),
event_normalizer=self._make_event_normalizer(),
usage_meter=self._make_usage_meter(),
translation_session=realtime_mode == "translation",
drop_params=drop_params,
)
await realtime_streaming.bidirectional_forward()

12
litellm/llms/openai/realtime/http_transformation.py
View file

@ -28,6 +28,18 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
base = base.removesuffix("/v1")
return join_api_base(base, "v1/realtime/transcription_sessions")
def get_translation_client_secret_url(
self, api_base: str | None, model: str, api_version: str | None = None
) -> str:
base = self.get_api_base(api_base).rstrip("/")
base = base.removesuffix("/v1")
return f"{base}/v1/realtime/translations/client_secrets"
def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base = self.get_api_base(api_base).rstrip("/")
base = base.removesuffix("/v1")
return f"{base}/v1/realtime/translations/calls"
def validate_environment(
self,
headers: dict,

2
litellm/llms/openai/responses/transformation.py
View file

@ -12,6 +12,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
deployment_model_info,
deployment_prompt_cache_breakpoint_support,
targets_openai_api,
)
@ -767,6 +768,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
custom_llm_provider=_provider_name(self.custom_llm_provider),
api_base=host.api_base or host.base_url,
prompt_cache_options=prompt_cache_options,
model_info=deployment_model_info(litellm_params),
)
@staticmethod

15
litellm/llms/openai/transcriptions/gpt_transformation.py
View file

@ -26,8 +26,8 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
self,
model: str,
audio_file: FileTypes,
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
) -> AudioTranscriptionRequestData:
"""
Transform the audio transcription request
@ -37,3 +37,14 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
return AudioTranscriptionRequestData(
data=data,
)
class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig):
def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]:
return [
"prompt",
"response_format",
"keywords",
"languages",
"stream",
]

42
litellm/llms/openai/transcriptions/handler.py
View file

@ -1,8 +1,9 @@
from collections.abc import Mapping
from collections.abc import Coroutine, Mapping
from typing import TYPE_CHECKING, Final, Optional, cast
import httpx
from openai import AsyncOpenAI, OpenAI
from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream
from openai.types.audio import TranscriptionStreamEvent
from pydantic import BaseModel
if TYPE_CHECKING:
@ -12,6 +13,9 @@ from litellm.litellm_core_utils.core_helpers import set_provider_response_header
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.audio_transcription.transformation import (
BaseAudioTranscriptionConfig,
acreate_sdk_transcription,
create_sdk_transcription,
sdk_compatible_transcription_request_data,
)
from litellm.types.utils import FileTypes
from litellm.utils import (
@ -28,31 +32,18 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
async def make_openai_audio_transcriptions_request(
self,
openai_aclient: AsyncOpenAI,
data: dict,
data: Mapping[str, object],
timeout: float | httpx.Timeout,
):
try:
raw_response = await openai_aclient.audio.transcriptions.with_raw_response.create(**data, timeout=timeout)
headers: Final = dict(raw_response.headers)
response: Final = raw_response.parse()
return headers, response
except Exception as e:
raise e
return await acreate_sdk_transcription(openai_aclient, sdk_compatible_transcription_request_data(data), timeout)
def make_sync_openai_audio_transcriptions_request(
self,
openai_client: OpenAI,
data: dict,
data: Mapping[str, object],
timeout: float | httpx.Timeout,
):
try:
raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**data, timeout=timeout)
headers: Final = dict(raw_response.headers)
response: Final = raw_response.parse()
return headers, response
except Exception as e:
raise e
return create_sdk_transcription(openai_client, sdk_compatible_transcription_request_data(data), timeout)
def audio_transcriptions(
self,
@ -70,7 +61,11 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
atranscription: bool = False,
provider_config: BaseAudioTranscriptionConfig | None = None,
shared_session: Optional["ClientSession"] = None,
) -> TranscriptionResponse:
) -> (
TranscriptionResponse
| Stream[TranscriptionStreamEvent]
| Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
):
"""
Handle audio transcription request
"""
@ -128,6 +123,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
)
logging_obj.model_call_details["response_headers"] = headers
if isinstance(response, Stream):
return response
if isinstance(response, BaseModel):
stringified_response = response.model_dump()
else:
@ -153,7 +151,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
async def async_audio_transcriptions(
self,
audio_file: FileTypes,
data: dict,
data: Mapping[str, object],
model_response: TranscriptionResponse,
timeout: float,
logging_obj: LiteLLMLoggingObj,
@ -192,6 +190,8 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
timeout=timeout,
)
logging_obj.model_call_details["response_headers"] = headers
if isinstance(response, AsyncStream):
return response
if isinstance(response, BaseModel):
stringified_response = response.model_dump()
else:

3
litellm/llms/opencode/chat/transformation.py
View file

@ -112,6 +112,9 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig):
def custom_llm_provider(self) -> str:
return "opencode_go"
def translates_prompt_cache_breakpoint(self, model: str) -> bool:
return "claude" in model.lower()
def get_complete_url(
self,
api_base: str | None,

123
litellm/main.py
View file

@ -37,6 +37,8 @@ if TYPE_CHECKING:
import dotenv
import httpx
import openai
from openai import AsyncStream, Stream
from openai.types.audio import TranscriptionStreamEvent
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import assert_never, overload
@ -753,6 +755,7 @@ async def acompletion(
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
routed_model_info,
)
from litellm.types.llms.openai import AllMessageValues
@ -764,6 +767,7 @@ async def acompletion(
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=base_url or kwargs.get("api_base"),
model_info=routed_model_info(caller_kwargs.get("model_info")),
)
prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs
@ -1001,7 +1005,7 @@ def _model_response_from_cached_dict(cached_response_dict: Mapping[str, object])
def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TranscriptionResponse:
return TranscriptionResponse(**cached_response_dict)
return TranscriptionResponse.model_validate(cached_response_dict)
def _text_completion_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TextCompletionResponse:
@ -5825,6 +5829,7 @@ def completion(
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
routed_model_info,
)
from litellm.types.llms.openai import AllMessageValues
@ -5836,6 +5841,7 @@ def completion(
tools=tools,
enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs
api_base=base_url or kwargs.get("api_base"),
model_info=routed_model_info(cast(object, model_info)), # cast-ok: untyped kwargs
)
if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and (
@ -8501,7 +8507,10 @@ async def amoderation(
@client
async def atranscription(*args, **kwargs) -> TranscriptionResponse:
async def atranscription(
*args, # noqa: ANN002 # public SDK wrapper preserves positional call compatibility
**kwargs, # noqa: ANN003 # kwargs-ok: public SDK wrapper preserves keyword call compatibility
) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]:
"""
Calls openai + azure whisper endpoints.
@ -8545,6 +8554,14 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
else:
# Call the synchronous function using run_in_executor
response = await loop.run_in_executor(None, func_with_context)
if isinstance(response, AsyncStream):
if file is not None:
calculated_duration = calculate_request_duration(file)
if calculated_duration is not None:
response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it
calculated_duration
)
return response
if not isinstance(response, TranscriptionResponse):
raise ValueError(
f"Invalid response from transcription provider, expected TranscriptionResponse, but got {type(response)}"
@ -8557,9 +8574,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
if response is not None and not isinstance(response, Coroutine) and file is not None:
existing_duration: Final = getattr(response, "duration", None)
if existing_duration is None:
calculated_duration: Final = calculate_request_duration(file)
if calculated_duration is not None:
response.hidden_params["audio_transcription_duration"] = calculated_duration
sync_calculated_duration: Final = calculate_request_duration(file)
if sync_calculated_duration is not None:
response.set_audio_transcription_duration(sync_calculated_duration)
return response
except Exception as e:
@ -8573,6 +8590,42 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse:
)
def _validate_gpt_transcription_request(
model: str,
pricing_model: str,
custom_llm_provider: str,
response_format: str | None,
drop_params: object,
) -> str | None:
model_cost_key: Final = next(
(
key
for key in (f"{custom_llm_provider}/{pricing_model}", pricing_model)
if (as_str_mapping(litellm.model_cost.get(key)) or {}).get("litellm_provider") == custom_llm_provider
),
None,
)
model_info: Final = (
get_model_info(model=pricing_model, custom_llm_provider=custom_llm_provider)
if model_cost_key is not None
else None
)
provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None
if (
provider_specific_entry is None
or provider_specific_entry.get("transcription_json_or_text_only") != 1
or response_format in (None, "json", "text")
):
return response_format
if litellm.drop_params is True or normalize_drop_params(drop_params) is True:
return None
raise litellm.UnsupportedParamsError(
message=f"{model} only supports response_format 'json' or 'text'",
model=model,
llm_provider=custom_llm_provider,
)
@client
def transcription(
model: str,
@ -8592,8 +8645,15 @@ def transcription(
max_retries: int | None = None,
custom_llm_provider=None,
base_url: str | None = None,
languages: Sequence[str] | None = None,
keywords: Sequence[str] | None = None,
stream: bool | None = None,
**kwargs,
) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]:
) -> (
TranscriptionResponse
| Stream[TranscriptionStreamEvent]
| Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
):
"""
Calls openai + azure whisper endpoints.
@ -8631,19 +8691,36 @@ def transcription(
api_key = dynamic_api_key if dynamic_api_key is not None else api_key
litellm_params_dict: Final = get_litellm_params(**kwargs)
configured_base_model: Final = litellm_params_dict.get("base_model")
pricing_model: Final = (
configured_base_model.removeprefix(f"{custom_llm_provider}/")
if isinstance(configured_base_model, str)
else model
)
supported_response_format: Final = _validate_gpt_transcription_request(
model=model,
pricing_model=pricing_model,
custom_llm_provider=custom_llm_provider,
response_format=response_format,
drop_params=kwargs.get("drop_params"),
)
optional_params: Final = get_optional_params_transcription(
model=model,
language=language,
languages=languages,
keywords=keywords,
prompt=prompt,
response_format=response_format,
response_format=supported_response_format,
timestamp_granularities=timestamp_granularities,
temperature=temperature,
stream=stream,
custom_llm_provider=custom_llm_provider,
**non_default_params,
)
litellm_params_dict: Final = get_litellm_params(**kwargs)
litellm_logging_obj.update_environment_variables(
model=model,
user=user,
@ -8660,11 +8737,17 @@ def transcription(
custom_llm_provider=custom_llm_provider,
)
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]
response: (
TranscriptionResponse
| Stream[TranscriptionStreamEvent]
| AsyncStream[TranscriptionStreamEvent]
| Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]]
)
provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model,
provider=LlmProviders(custom_llm_provider),
base_model=pricing_model,
)
uses_openai_transport: Final = custom_llm_provider in OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS and not (
@ -8675,7 +8758,7 @@ def transcription(
# azure configs
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
azure_api_version: Final = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN")
@ -8694,7 +8777,7 @@ def transcription(
logging_obj=litellm_logging_obj,
api_base=api_base,
api_key=api_key,
api_version=api_version,
api_version=azure_api_version,
azure_ad_token=azure_ad_token,
max_retries=max_retries,
litellm_params=litellm_params_dict,
@ -8831,15 +8914,17 @@ def transcription(
# Store duration in _hidden_params for cost calculation without
# exposing it in the response body (see sync path comment above).
if not isinstance(response, Coroutine):
existing_duration: Final = getattr(response, "duration", None)
if existing_duration is None:
calculated_duration: Final = calculate_request_duration(file)
if not isinstance(response, Coroutine) and (
isinstance(response, (Stream, AsyncStream)) or getattr(response, "duration", None) is None
):
calculated_duration: Final = calculate_request_duration(file)
if isinstance(response, (Stream, AsyncStream)):
if calculated_duration is not None:
response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior
dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR)
response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it
calculated_duration
)
response_hidden_params["audio_transcription_duration"] = calculated_duration
elif calculated_duration is not None:
response.set_audio_transcription_duration(calculated_duration)
return response

266
litellm/model_prices_and_context_window_backup.json
View file

@ -1,4 +1,168 @@
{
"azure/gpt-realtime-2.1-2026-07-07": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
"input_cost_per_token": 4e-06,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"text",
"image",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"cache_read_input_image_token_cost": 5e-07,
"supports_prompt_caching": true,
"supports_reasoning": true
},
"azure/gpt-realtime-2.1-mini-2026-07-07": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"text",
"image",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"cache_read_input_image_token_cost": 8e-08,
"supports_prompt_caching": true,
"supports_reasoning": true
},
"azure/gpt-realtime-translate-2026-05-06": {
"input_cost_per_second": 0.0005666666666666667,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"audio",
"text"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_native_streaming": true
},
"azure/gpt-realtime-translate-2026-05-07": {
"input_cost_per_second": 0.0005666666666666667,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://ai.azure.com/catalog/models/gpt-realtime-translate",
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"audio",
"text"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_native_streaming": true
},
"azure/gpt-realtime-whisper-2026-05-06": {
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
"/v1/realtime/transcription_sessions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"max_input_tokens": 32000,
"max_output_tokens": 4096
},
"azure/gpt-realtime-whisper-2026-05-07": {
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
"/v1/realtime/transcription_sessions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"max_input_tokens": 32000,
"max_output_tokens": 4096
},
"sample_spec": {
"code_interpreter_cost_per_session": 0.0,
"computer_use_input_cost_per_1k_tokens": 0.0,
@ -6424,6 +6588,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6458,6 +6625,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6491,6 +6661,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6525,6 +6698,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6547,6 +6723,7 @@
"azure/gpt-realtime-2.1": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_image_token_cost": 5e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-06-25",
"input_cost_per_audio_token": 3.2e-05,
@ -6556,6 +6733,9 @@
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
@ -6575,12 +6755,15 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"azure/gpt-realtime-2.1-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_image_token_cost": 8e-08,
"cache_read_input_token_cost": 6e-08,
"deprecation_date": "2027-06-25",
"input_cost_per_audio_token": 1e-05,
@ -6590,6 +6773,9 @@
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
@ -6609,6 +6795,8 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
@ -6626,6 +6814,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6659,6 +6850,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6791,6 +6985,9 @@
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -29238,9 +29435,9 @@
},
"gemini-3-pro-image": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_flex": 1e-07,
"cache_read_input_token_cost_priority": 3.6e-07,
"deprecation_date": "2027-05-28",
@ -39069,6 +39266,9 @@
"max_input_tokens": 128000,
"max_output_tokens": 32000,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://developers.openai.com/api/docs/pricing",
@ -39124,6 +39324,8 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
@ -39160,6 +39362,8 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
@ -43106,18 +43310,18 @@
"mode": "embedding"
},
"mistral/mistral-large-4": {
"cache_read_input_token_cost": 1.4e-07,
"input_cost_per_token": 1.36e-06,
"cache_read_input_token_cost": 7e-08,
"input_cost_per_token": 7e-07,
"litellm_provider": "mistral",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 4.18e-06,
"output_cost_per_token": 2.1e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/mistral-large-4",
"source": "https://docs.mistral.ai/models/mistral-large-4-0",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -43127,18 +43331,18 @@
"supports_vision": true
},
"mistral/mistral-large-4-0": {
"cache_read_input_token_cost": 1.4e-07,
"input_cost_per_token": 1.36e-06,
"cache_read_input_token_cost": 7e-08,
"input_cost_per_token": 7e-07,
"litellm_provider": "mistral",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 4.18e-06,
"output_cost_per_token": 2.1e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/mistral-large-4",
"source": "https://docs.mistral.ai/models/mistral-large-4-0",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -52587,9 +52791,9 @@
},
"vertex_ai/gemini-3-pro-image": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_flex": 1e-07,
"cache_read_input_token_cost_priority": 3.6e-07,
"deprecation_date": "2027-05-28",
@ -63405,6 +63609,10 @@
"input_cost_per_second": 7.5e-05,
"litellm_provider": "openai",
"mode": "audio_transcription",
"provider_specific_entry": {
"transcription_json_or_text_only": 1,
"transcription_usage_duration": 1
},
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/audio/transcriptions",
@ -63417,7 +63625,8 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"supports_native_streaming": true
},
"gpt-live-transcribe": {
"input_cost_per_second": 0.000283333333333,
@ -63435,7 +63644,8 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"supports_native_streaming": true
},
"gpt-live-1": {
"input_cost_per_second": 0.000833333333333,
@ -63469,7 +63679,8 @@
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true
"supports_audio_output": true,
"supports_native_streaming": true
},
"claude-mythos-5": {
"supports_anthropic_compaction": true,
@ -71340,6 +71551,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/realtime"
@ -71397,7 +71611,11 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supports_native_streaming": true
},
"azure/gpt-transcribe": {
"deprecation_date": "2028-02-01",
@ -71407,7 +71625,8 @@
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/audio/transcriptions",
"/v1/realtime/transcription_sessions"
"/v1/realtime/transcription_sessions",
"/v1/realtime"
],
"supported_modalities": [
"text",
@ -71416,7 +71635,13 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"provider_specific_entry": {
"realtime_ga_only": 1,
"transcription_json_or_text_only": 1,
"transcription_usage_duration": 1
},
"supports_native_streaming": true
},
"azure/gpt-realtime-translate": {
"deprecation_date": "2027-11-09",
@ -71434,7 +71659,11 @@
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true
"supports_audio_output": true,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supports_native_streaming": true
},
"aihubmix/agnes-2.5-flash": {
"input_cost_per_token": 3e-08,
@ -72873,7 +73102,8 @@
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"typesafe/jev-preview": {
"input_cost_per_token": 4.2e-08,

7
litellm/proxy/_lazy_features.py
View file

@ -504,10 +504,13 @@ def _lazy_lock(app: "FastAPI", module_path: str) -> asyncio.Lock:
def _register_feature(app: "FastAPI", feat: LazyFeature, module: object, features: tuple[LazyFeature, ...]) -> None:
before: Final = len(app.router.routes)
routes_before: Final = frozenset(id(route) for route in app.router.routes)
feat.register_fn(app, module)
lazy_routes: Final[Mapping[str, tuple[BaseRoute, ...]]] = MappingProxyType(
{**_lazy_routes(app), feat.module_path: tuple(app.router.routes[before:])}
{
**_lazy_routes(app),
feat.module_path: tuple(route for route in app.router.routes if id(route) not in routes_before),
}
)
app.state.lazy_routes = lazy_routes # rebind-ok: the app owns the record of which routes each feature added
app.router.routes[:] = _route_table( # rebind-ok: the app owns its route table

968
litellm/proxy/_lazy_openapi_snapshot.json
View file

File diff suppressed because it is too large Load diff

6
litellm/proxy/_types.py
View file

@ -332,6 +332,8 @@ class LiteLLMRoutes(enum.Enum):
"/realtime?{model}",
"/v1/realtime?{model}",
"/openai/v1/realtime?{model}",
"/realtime/translations",
"/v1/realtime/translations",
# realtime (GA WebRTC HTTP routes)
"/realtime/client_secrets",
"/v1/realtime/client_secrets",
@ -342,6 +344,10 @@ class LiteLLMRoutes(enum.Enum):
"/realtime/transcription_sessions",
"/v1/realtime/transcription_sessions",
"/openai/v1/realtime/transcription_sessions",
"/realtime/translations/client_secrets",
"/v1/realtime/translations/client_secrets",
"/realtime/translations/calls",
"/v1/realtime/translations/calls",
# responses API
"/responses",
"/v1/responses",

16
litellm/proxy/agent_endpoints/auth/managed_authorization.py
View file

@ -11,7 +11,15 @@ from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failur
from litellm.types.agents import AgentResponse
from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext
_MANAGED_REALTIME_ROUTES: Final = frozenset(("/realtime", "/v1/realtime", "/openai/v1/realtime"))
_MANAGED_REALTIME_ROUTES: Final = frozenset(
(
"/realtime",
"/v1/realtime",
"/openai/v1/realtime",
"/realtime/translations",
"/v1/realtime/translations",
)
)
_MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions"))
_MANAGED_MODEL_ROUTES: Final = frozenset(
f"{prefix}/{operation}"
@ -112,7 +120,11 @@ def managed_inference_request(
if route in _MANAGED_LIVE_ROUTES:
return dict(body)
if route in _MANAGED_REALTIME_ROUTES:
model: Final = query_model or body.get("model")
from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model
model: Final = resolve_realtime_route_model(
query_model if isinstance(query_model, str) else None, None, route.endswith("/realtime/translations")
)
if not isinstance(model, str) or not model:
raise_identity_failure(
AgentIdentityFailure(message="Managed inference requires an explicit or configured model")

1
litellm/proxy/auth/auth_utils.py
View file

@ -1971,6 +1971,7 @@ _MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS: Final = ("/evals",)
# can_key_call_model() validates the real target model.
_MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS: Final = (
"/realtime/client_secrets",
"/realtime/translations/client_secrets",
"/realtime/calls",
)
_MODEL_ROUTING_ID_FIELDS: Final = (

113
litellm/proxy/auth/model_checks.py
View file

@ -23,7 +23,12 @@ from litellm.router import Router
from litellm.router_api import fallback_chain
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
from litellm.utils import (
AZURE_MODEL_PLACEHOLDER,
ProviderConfigManager,
get_valid_models,
potential_model_names_in_lookup_order,
)
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value})
@ -375,6 +380,10 @@ def _ids_routing_to_catalog_models(
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
if name_pattern.count("*") > deployment_pattern.count("*"):
return _catalog_ids_a_wider_name_matches(
name_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments(
name_pattern, deployment_pattern
):
@ -419,6 +428,23 @@ def _ids_routing_to_catalog_models(
)
def _catalog_ids_a_wider_name_matches(
name_pattern: str,
deployment_pattern: str,
provider: str,
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
name_regex: Final = _wildcard_name_regex(name_pattern)
return tuple(
catalog_id
for catalog_id in _ids_routing_to_catalog_models(
deployment_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_regex.fullmatch(catalog_id)
)
def _model_info_field(deployment: Mapping[str, object], field: str) -> object:
model_info: Final = deployment.get("model_info")
return model_info.get(field) if is_str_keyed_mapping(model_info) else None
@ -456,11 +482,12 @@ def _routes_to_model(
return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id)
def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None:
def _deployment_pattern_naming_listed_ids(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model):
return None
name_suffix: Final = wildcard_model.split("/", 1)[-1]
deployment_suffix: Final = litellm_params.model.partition("/")[2]
return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None
return litellm_params.model if name_suffix != "*" or deployment_suffix != "*" else None
def get_known_models_from_wildcard(
@ -468,6 +495,43 @@ def get_known_models_from_wildcard(
litellm_params: LiteLLM_Params | None = None,
routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True,
) -> list[str]:
provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params)
if litellm_params is None or provider_id_prefix is None:
return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model))
return [
listed_id.removeprefix(provider_id_prefix)
for listed_id in _known_models_from_wildcard(
wildcard_model,
litellm_params.model_copy(update={"model": f"{provider_id_prefix}{litellm_params.model}"}),
lambda listed_id, model: routes_to_model(
listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix)
),
)
]
def _custom_llm_provider_prefix(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if (
litellm_params is None
or not litellm_params.custom_llm_provider
or "/" in wildcard_model
or wildcard_model == "*"
or "/" in litellm_params.model
or not _check_wildcard_routing(litellm_params.model)
):
return None
return f"{litellm_params.custom_llm_provider}/"
def _without_azure_placeholder(catalog_ids: Sequence[str]) -> tuple[str, ...]:
return tuple(catalog_id for catalog_id in catalog_ids if catalog_id != AZURE_MODEL_PLACEHOLDER)
def _known_models_from_wildcard(
wildcard_model: str,
litellm_params: LiteLLM_Params | None,
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
wildcard_model_to_expand: Final = (
litellm_params.model
if wildcard_model == "*"
@ -476,15 +540,14 @@ def get_known_models_from_wildcard(
and "/" in litellm_params.model
else wildcard_model
)
try:
wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1)
except ValueError: # safely fail
return []
deployment_pattern_naming_listed_ids: Final = (
wildcard_model_to_expand
if wildcard_model_to_expand != wildcard_model
else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
else _deployment_pattern_naming_listed_ids(wildcard_model, litellm_params)
)
wildcard_provider_prefix, has_provider_prefix, wildcard_suffix = wildcard_model_to_expand.partition("/")
if not has_provider_prefix and deployment_pattern_naming_listed_ids is None:
return ()
# Use provider from litellm_params when available, otherwise from wildcard prefix
# (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router)
@ -501,16 +564,14 @@ def get_known_models_from_wildcard(
wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params)
if wildcard_models is None:
return []
return ()
if deployment_pattern_naming_listed_ids is not None:
return list(
_ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models,
routes_to_model=routes_to_model,
)
return _ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models if has_provider_prefix else _without_azure_placeholder(wildcard_models),
routes_to_model=routes_to_model,
)
if wildcard_suffix != "*":
model_prefix: Final = wildcard_suffix.replace("*", "")
@ -526,7 +587,7 @@ def get_known_models_from_wildcard(
# add model prefix to wildcard models
wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models]
return list(
return tuple(
dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models)
)
@ -564,16 +625,6 @@ def _names_no_deployment_serves_by_name(
return tuple(name for name in names if not deployed_by_name(name, team_id))
def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None:
if not _check_wildcard_routing(model_name):
return None
if "/" in model_name or model_name == "*":
return model_name
if _check_wildcard_routing(litellm_model):
return litellm_model
return model_name
def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool:
if allowed_model_names is None:
return True
@ -601,10 +652,10 @@ def expand_wildcard_deployments_for_model_info(
raw_params = deployment.get("litellm_params")
litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {}
litellm_model = str(litellm_params_dict.get("model") or "")
if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None:
if not _check_wildcard_routing(model_name):
expanded.append(deployment)
continue
if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names):
if not _can_expand_to_an_allowed_name(model_name, allowed_model_names):
continue
try:
@ -613,7 +664,7 @@ def expand_wildcard_deployments_for_model_info(
expanded.append(deployment)
continue
expanded_names = get_known_models_from_wildcard(
wildcard_model=wildcard_pattern,
wildcard_model=model_name,
litellm_params=litellm_params,
routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)),
)

17
litellm/proxy/auth/user_api_key_auth.py
View file

@ -148,6 +148,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg
populate_request_with_path_params,
read_raw_json_body,
read_request_body,
resolve_realtime_route_model,
rewrite_request_model,
safe_get_request_headers,
safe_get_request_query_params,
@ -1246,11 +1247,21 @@ def _apply_budget_limits_to_end_user_params(
async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth:
return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model"))
return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model", ""))
async def user_api_key_auth_realtime_websocket(websocket: WebSocket) -> UserAPIKeyAuth:
route_path: Final[object] = websocket.scope.get("path")
model: Final = resolve_realtime_route_model(
model=websocket.query_params.get("model"),
intent=websocket.query_params.get("intent"),
is_translation=isinstance(route_path, str) and route_path.endswith("/realtime/translations"),
)
return await user_api_key_auth_websocket_for_model(websocket, model=model or "")
async def user_api_key_auth_live_websocket(websocket: WebSocket) -> UserAPIKeyAuth:
return await user_api_key_auth_websocket_for_model(websocket, model=None)
return await user_api_key_auth_websocket_for_model(websocket, model="")
def _has_required_baggage_tag_mapping() -> bool:
@ -1266,7 +1277,7 @@ def _has_required_baggage_tag_mapping() -> bool:
async def user_api_key_auth_websocket_for_model(
websocket: WebSocket, model: str | None, *, close_on_auth_error: bool = True
websocket: WebSocket, model: str, *, close_on_auth_error: bool = True
) -> UserAPIKeyAuth:
ws_scope: Final = websocket.scope or {}
scope_headers: Final = list(ws_scope.get("headers") or [])

2
litellm/proxy/common_request_processing.py
View file

@ -171,7 +171,9 @@ ProxyRouteType: TypeAlias = Literal[
"_arealtime",
"_aresponses_websocket",
"acreate_realtime_client_secret",
"acreate_realtime_translation_client_secret",
"arealtime_calls",
"arealtime_translation_calls",
"aget_responses",
"adelete_responses",
"acancel_responses",

28
litellm/proxy/common_utils/http_parsing_utils.py
View file

@ -37,6 +37,16 @@ _DECODED_JSON: Final = TypeAdapter(object)
_JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object])
def resolve_realtime_route_model(model: str | None, intent: str | None, is_translation: bool) -> str | None:
if model is not None:
return model
if is_translation:
return "gpt-realtime-translate"
if intent == "transcription":
return "gpt-realtime-whisper"
return None
def resolve_inference_model(
body_model: object,
settings: Mapping[str, object],
@ -572,14 +582,16 @@ async def get_form_data(request: Request) -> dict[str, Any]:
Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]`
"""
form: Final = await request.form()
parsed_form_data: Final[dict[str, Any]] = {}
for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat
if key.endswith("[]"):
clean_key = key[:-2]
parsed_form_data.setdefault(clean_key, []).append(value)
else:
parsed_form_data[key] = value
return parsed_form_data
form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items())
array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]"))
normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items)
normalized_keys: Final = tuple(dict.fromkeys(key for key, _ in normalized_items))
return {
key: [value for item_key, value in normalized_items if item_key == key]
if key in array_keys
else next(value for item_key, value in reversed(normalized_items) if item_key == key)
for key in normalized_keys
}
async def convert_upload_files_to_file_data(

2
litellm/proxy/common_utils/realtime_utils.py
View file

@ -11,7 +11,7 @@ def realtime_request_body(model: str | None) -> bytes:
Generate the realtime websocket request body. Cached with LRU semantics to avoid repeated
string formatting work while keeping memory usage bounded.
"""
return json.dumps({"model": model or ""}).encode()
return json.dumps({"model": model}).encode()
_realtime_request_body: Final = realtime_request_body

92
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -709,7 +709,7 @@ class RequestRateLimiterStash:
A single mutable instance is shared by every context forked from the
request task (the SDK call, streaming generators, and the logging worker's
captured context all see the same object), which is what makes the
``reservation_released`` flag and ``parallel_slot`` clearing effective
released flags and ``parallel_slot`` clearing effective
across sibling callbacks: the first release wins, later callbacks observe
the cleared state.
@ -746,9 +746,32 @@ class RequestRateLimiterStash:
batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None
batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
reservation_released: bool = False
io_reservation_released: bool = False
tpm_limited_tags: frozenset[str] = field(default_factory=frozenset)
socket_agent_windows: tuple[RateLimitDescriptor, ...] = ()
def clear_reserved_tokens(self) -> None:
self.reserved_tokens = 0
self.ptu_reserved_tokens = 0
self.itpm_reserved_tokens = 0
self.otpm_reserved_tokens = 0
def mark_reservation_released(self) -> None:
self.reservation_released = True
self.io_reservation_released = True
def discard_released_tpm_reservation(self) -> None:
if self.reservation_released:
self.reserved_tokens = 0
self.ptu_reserved_tokens = 0
self.reservation_released = False
def discard_released_io_reservation(self) -> None:
if self.io_reservation_released:
self.itpm_reserved_tokens = 0
self.otpm_reserved_tokens = 0
self.io_reservation_released = False
@dataclass(frozen=True, slots=True)
class CounterRefund:
@ -4238,6 +4261,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
requested_model=requested_model,
)
stash.discard_released_io_reservation()
if itpm_reserved > 0:
itpm_scopes: Final = tuple(
(d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY
@ -4597,6 +4621,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# incremented so post-call reconciliation only applies
# the (actual - reserved) delta to those — unreserved
# scopes get charged the full actual usage instead.
stash.discard_released_tpm_reservation()
stash.reserved_tokens = estimated_tokens
stash.ptu_reserved_tokens = ptu_estimated_tokens
stash.ptu_reserved_window_identities = frozenset(
@ -5140,7 +5165,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._response_usage(response_obj)
) or self._aggregate_only_total_tokens(self._response_usage(kwargs.get("combined_usage_object")))
if not response_usage[2] and not combined_usage[2] and aggregate_total <= 0 and not stash.reservation_released:
if (
not response_usage[2]
and not combined_usage[2]
and aggregate_total <= 0
and not stash.io_reservation_released
):
return ()
resolved_usage: Final = (
response_usage
@ -5153,25 +5183,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
billable_input, completion_tokens, _ = resolved_usage
if stash.reservation_released or (
if stash.io_reservation_released or (
not stash.itpm_reserved_window_identities and not stash.otpm_reserved_window_identities
):
return self._build_reservation_aware_tpm_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes,
actual_tokens=billable_input,
reserved_tokens=0 if stash.reservation_released else itpm_reserved,
reserved_tokens=0 if stash.io_reservation_released else itpm_reserved,
) + self._build_reservation_aware_tpm_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes,
actual_tokens=completion_tokens,
reserved_tokens=0 if stash.reservation_released else otpm_reserved,
reserved_tokens=0 if stash.io_reservation_released else otpm_reserved,
)
itpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = (
self._build_project_reservation_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes,
actual_tokens=billable_input,
reserved_tokens=itpm_reserved,
reservation_window_identities=stash.itpm_reserved_window_identities,
@ -5182,7 +5212,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
otpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = (
self._build_project_reservation_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes,
actual_tokens=completion_tokens,
reserved_tokens=otpm_reserved,
reservation_window_identities=stash.otpm_reserved_window_identities,
@ -5402,7 +5432,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
total_tokens = self._aggregate_only_total_tokens(usage=_usage)
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
reserved_tokens: Final = stash.reserved_tokens if stash is not None else 0
open_reservation: Final = stash if stash is not None and not stash.reservation_released else None
reserved_tokens: Final = open_reservation.reserved_tokens if open_reservation is not None else 0
reserved_model: Final = stash.reserved_model if stash is not None else None
reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset()
# Reconciliation must target the same model-scoped counter that the
@ -5464,7 +5495,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
reconcile_model=reconcile_model,
reserved_scopes=reserved_scopes,
reserved_ceiling=stash.ptu_ceiling if stash is not None else None,
reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0,
reserved_tokens=open_reservation.ptu_reserved_tokens if open_reservation is not None else 0,
reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(),
total_tokens=0 if ptu_skips_socket_usage else total_tokens,
)
@ -5561,18 +5592,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else None,
)
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
),
litellm_parent_otel_span,
usage_operations: Final = self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
)
io_token_operations: Final = self._build_io_token_reservation_ops(
kwargs=kwargs,
response_obj=response_obj,
)
if stash is not None:
stash.clear_reserved_tokens()
await self._apply_usage_operations(usage_operations, litellm_parent_otel_span)
if io_token_operations:
if isinstance(io_token_operations, list):
await self.async_increment_tokens_with_ttl_preservation(
@ -5733,10 +5764,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# already released it (proxy-level rejection that also bubbles up
# here as an LLM-error callback). max_parallel_requests is its
# own counter and is always decremented per call.
reserved_tokens, itpm_reserved, otpm_reserved = (
(0, 0, 0)
if stash is None or stash.reservation_released
else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
reserved_tokens: Final = 0 if stash is None or stash.reservation_released else stash.reserved_tokens
itpm_reserved, otpm_reserved = (
(0, 0)
if stash is None or stash.io_reservation_released
else (stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
)
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call)
@ -5806,6 +5838,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else ()
)
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.mark_reservation_released()
if pipeline_operations:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline_post_call(
pipeline_operations, parent_otel_span=litellm_parent_otel_span
@ -5820,8 +5854,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
pipeline_operations=project_operations,
parent_otel_span=litellm_parent_otel_span,
)
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.reservation_released = True
if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call):
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
@ -5927,7 +5959,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
Idempotent: the slot release clears the stashed acquisition (and slot
removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM
refund is guarded by the stash's ``reservation_released`` flag — if
refund is guarded by the stash's two released flags — if
both this hook and async_log_failure_event end up running in the same
flow, only the first release/refund applies. A mid-stream failure
relayed here with recovered partial usage settles the reservation at
@ -5953,11 +5985,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
stash.batch_tpd_refund_ops = ()
if stash.reservation_released:
return
reserved_tokens: Final = stash.reserved_tokens
itpm_reserved: Final = stash.itpm_reserved_tokens
otpm_reserved: Final = stash.otpm_reserved_tokens
reserved_tokens: Final = 0 if stash.reservation_released else stash.reserved_tokens
itpm_reserved: Final = 0 if stash.io_reservation_released else stash.itpm_reserved_tokens
otpm_reserved: Final = 0 if stash.io_reservation_released else stash.otpm_reserved_tokens
if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0:
return
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(request_data)
@ -6024,6 +6054,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
itpm_reserved,
otpm_reserved,
)
stash.mark_reservation_released()
if combined_ops:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
increment_list=list(combined_ops),
@ -6040,7 +6071,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
pipeline_operations=project_ops,
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash.reservation_released = True
except Exception as e:
verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e)
return

10
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -64,7 +64,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
_sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
deployment_params_provider,
get_request_model_access_groups,
sanitize_error_information_for_spend_logs,
sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export
should_store_prompts_and_responses_in_spend_logs,
)
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs
@ -79,7 +79,6 @@ from litellm.types.utils import (
CallTypes,
LiteLLMBatch,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
from litellm.utils import get_end_user_id_for_cost_tracking
@ -492,13 +491,6 @@ class _ProxyDBLogger(CustomLogger):
# ``.get("traceback")`` / truthy checks, and the TypedDict marks
# the field as optional, so omitting is type-safe.
_error_information.pop("traceback", None)
# Strip echoed request input + apply DB-size cap before storing in
# the spend-log metadata column (LIT-2992). Result is never None
# here because the input above is constructed non-None.
_error_information = cast(
StandardLoggingPayloadErrorInformation,
sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception),
)
_metadata["error_information"] = _error_information
_metadata = ( # rebind-ok: enriched identity is shared with the earlier immutable capture

176
litellm/proxy/proxy_server.py
View file

@ -60,6 +60,7 @@ import anyio
import anyio.lowlevel
import websockets
import websockets.exceptions
from openai.types.audio import TranscriptionStreamEvent
from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config
from pydantic.fields import FieldInfo, PydanticUndefined
from pydantic.main import IncEx
@ -215,6 +216,7 @@ from litellm.types.utils import (
TextCompletionResponse,
TokenCountMessagesFormat,
TokenCountResponse,
TranscriptionResponse,
echoed_cost_map_pricing_fields,
is_server_derived_pricing_key,
pricing_override_fields,
@ -324,6 +326,7 @@ from litellm._internal_context import service_target, with_service_target
from litellm._logging import (
STARTUP_OUTPUT_SETTINGS,
print_startup_message,
redact_internal_details_from_client_message,
redact_string,
verbose_proxy_logger,
verbose_router_logger,
@ -373,6 +376,11 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
TRANSCRIPTION_STREAM_DONE_FRAME,
GuardedTranscriptionReplay,
guarded_transcription_events,
)
from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
@ -471,6 +479,7 @@ from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # lega
user_api_key_auth,
user_api_key_auth_accepted_websocket_for_model,
user_api_key_auth_live_websocket,
user_api_key_auth_realtime_websocket,
user_api_key_auth_websocket,
)
from litellm.proxy.batches_endpoints.endpoints import router as batches_router
@ -536,6 +545,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF10
resolve_inference_model,
safe_get_request_headers,
)
from litellm.proxy.common_utils.http_parsing_utils import (
resolve_realtime_route_model as _resolve_realtime_route_model,
)
from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket
from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations
from litellm.proxy.common_utils.model_listing_utils import (
@ -555,9 +567,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
from litellm.proxy.common_utils.openai_error_payload import (
LITELLM_CALL_ID_HEADER,
attribute_of,
error_status_code,
headers_with_litellm_call_id,
litellm_call_id_headers,
openai_error_param,
openai_error_type,
proxy_exception_for,
route_failure_proxy_exception,
with_litellm_call_id,
@ -1171,6 +1185,27 @@ server_root_path: Final = get_server_root_path()
_license_check = LicenseCheck()
premium_user: bool = _license_check.is_premium()
premium_user_data: Optional["EnterpriseLicenseData"] = _license_check.airgapped_license_data
def _set_premium_user(is_premium: bool) -> None:
global premium_user, premium_user_data
premium_user = is_premium # rebind-ok: config load can switch the license in use
premium_user_data = _license_check.airgapped_license_data # rebind-ok: tracks the license in use
def _use_config_license(config_license: str, env_license_is_premium: bool) -> bool:
env_license: Final = _license_check.license_str
env_license_data: Final = _license_check.airgapped_license_data
_license_check.license_str = config_license
if _license_check.is_premium():
return True
if not env_license_is_premium:
return False
_license_check.license_str = env_license
_license_check.airgapped_license_data = env_license_data
return True
global_max_parallel_request_retries_env: Final[str | None] = os.getenv("LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES")
proxy_state: Final = ProxyState()
SENSITIVE_DATA_MASKER: Final = SensitiveDataMasker()
@ -6672,7 +6707,6 @@ class ProxyConfig:
def _load_environment_variables(self, config: dict):
## ENVIRONMENT VARIABLES
global premium_user
environment_variables: Final = config.get("environment_variables", None)
if environment_variables:
for key, value in environment_variables.items():
@ -6712,7 +6746,7 @@ class ProxyConfig:
# check if litellm_license in general_settings
if "LITELLM_LICENSE" in environment_variables:
_license_check.license_str = os.getenv("LITELLM_LICENSE", None)
premium_user = _license_check.is_premium()
_set_premium_user(_license_check.is_premium())
def _set_config_environment_variable(self, key: str, value: str) -> None:
if key in os.environ and os.environ[key] != value:
@ -6764,7 +6798,6 @@ class ProxyConfig:
prompt_injection_detection_obj, \
redis_usage_cache, \
store_model_in_db, \
premium_user, \
open_telemetry_logger, \
health_check_details, \
proxy_batch_polling_interval, \
@ -6777,6 +6810,12 @@ class ProxyConfig:
self._load_environment_variables(config=config)
config_license: Final = _GENERAL_SETTINGS_VIEW.validate_python(
_GENERAL_SETTINGS_VIEW.validate_python(config).get("general_settings") or {}
).get("litellm_license")
if isinstance(config_license, str):
_set_premium_user(_use_config_license(config_license, env_license_is_premium=premium_user))
## Coordination Redis (before cache init, so the explicit block wins)
coordination_redis_cache: Final = self._init_coordination_redis(config=config)
if coordination_redis_cache is not None:
@ -7353,7 +7392,7 @@ class ProxyConfig:
allowed_ips: Final = general_settings.get("allowed_ips", None)
if allowed_ips is not None and premium_user is False:
raise ValueError(
"allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment."
"allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment."
)
## BUDGET RESCHEDULER ##
proxy_budget_rescheduler_min_time = general_settings.get(
@ -7409,11 +7448,6 @@ class ProxyConfig:
if general_settings.get("enforced_params") is not None and premium_user is not True:
raise ValueError("Trying to use `enforced_params`" + CommonProxyErrors.not_premium_user.value)
# check if litellm_license in general_settings
if "litellm_license" in general_settings:
_license_check.license_str = general_settings["litellm_license"]
premium_user = _license_check.is_premium()
router_params: Final[dict] = {
"cache_responses": litellm.cache is not None, # cache if user passed in cache values
}
@ -10715,6 +10749,26 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob
return _resolve
def _transcription_sse_error_frame(client_error: Exception) -> str:
status_code: Final = error_status_code(client_error, 500)
error_returned: Final = json.dumps(
{
"error": sse_error_payload(client_error)[1]
if isinstance(client_error, HTTPException)
else ProxyException(
message=redact_internal_details_from_client_message(
str(getattr(client_error, "message", client_error))
),
type=openai_error_type(client_error, status_code),
param=openai_error_param(client_error),
code=status_code,
).to_dict()
},
default=str,
)
return f"data: {error_returned}\n\n"
async def async_data_generator(
response,
user_api_key_dict: UserAPIKeyAuth,
@ -13751,11 +13805,13 @@ def _form_fields_without_file(form_data: Mapping[str, object]) -> Mapping[str, o
"/v1/audio/transcriptions",
dependencies=[Depends(user_api_key_auth)],
tags=["audio"],
responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}},
)
@router.post(
"/audio/transcriptions",
dependencies=[Depends(user_api_key_auth)],
tags=["audio"],
responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}},
)
async def audio_transcriptions(
request: Request,
@ -13843,34 +13899,72 @@ async def audio_transcriptions(
cache_key: Final = hidden_params.get("cache_key", None) or ""
api_base: Final = hidden_params.get("api_base", None) or ""
response_cost: Final = hidden_params.get("response_cost", None) or ""
response_call_id: Final = hidden_params.get("litellm_call_id", None) or ""
response_call_id: Final = hidden_params.get("litellm_call_id", None) or litellm_call_id
additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {}
stream_logging_obj: Final = getattr(response, "logging_obj", None)
fastapi_response.headers.update(
ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
model_id=model_id,
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
call_id=response_call_id,
request_data=data,
hidden_params=hidden_params,
**additional_headers,
)
custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=user_api_key_dict,
model_id=model_id,
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
call_id=response_call_id,
request_data=data,
hidden_params=hidden_params,
litellm_logging_obj=stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None,
**additional_headers,
)
# Call response headers hook (matches base_process_llm_request behavior)
callback_headers: Final = await proxy_logging_obj.post_call_response_headers_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=response,
response=response.response if isinstance(response, GuardedTranscriptionReplay) else response,
request_headers=dict(request.headers),
)
if callback_headers:
fastapi_response.headers.update(callback_headers)
response_headers: Final = {**custom_headers, **(callback_headers or {})}
if hasattr(response, "__aiter__"):
async def transcription_event_stream(
stream: AsyncIterator[TranscriptionStreamEvent],
) -> AsyncGenerator[str, None]:
try:
async for event in stream:
yield f"data: {event.model_dump_json(exclude_unset=True)}\n\n"
yield TRANSCRIPTION_STREAM_DONE_FRAME
except Exception as stream_error: # noqa: BLE001 # stream boundary: a mid-stream failure ends the SSE body as an error event
replacement: Final = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=stream_error,
request_data=(
{**data, "litellm_logging_obj": stream_logging_obj}
if isinstance(stream_logging_obj, LiteLLMLoggingObj)
else data
),
)
yield _transcription_sse_error_frame(stream_error if replacement is None else replacement)
yield TRANSCRIPTION_STREAM_DONE_FRAME
finally:
with anyio.CancelScope(shield=True):
close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None)
if callable(close):
close_result: Final = close()
if inspect.isawaitable(close_result):
await close_result
return StreamingResponse(
guarded_transcription_events(response.response)
if isinstance(response, GuardedTranscriptionReplay)
else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)),
media_type="text/event-stream",
headers=response_headers,
)
fastapi_response.headers.update(response_headers)
return response
except Exception as e:
@ -14019,6 +14113,8 @@ async def _reject_realtime_session(
@app.websocket("/openai/v1/realtime")
@app.websocket("/v1/realtime")
@app.websocket("/realtime")
@app.websocket("/v1/realtime/translations")
@app.websocket("/realtime/translations")
async def realtime_websocket_endpoint(
websocket: WebSocket,
model: str | None = fastapi.Query(None, description="The model to use for the websocket connection."),
@ -14027,7 +14123,7 @@ async def realtime_websocket_endpoint(
None,
description="Comma-separated list of guardrail names to apply to this request.",
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_realtime_websocket),
):
requested_protocols: Final = tuple(
protocol.strip()
@ -14035,15 +14131,13 @@ async def realtime_websocket_endpoint(
if protocol.strip()
)
route_model = model
is_translation: Final = websocket.url.path.endswith("/realtime/translations")
route_model: Final = _resolve_realtime_route_model(model, intent, is_translation)
if route_model is None:
if intent == "transcription":
route_model = "gpt-realtime-whisper"
else:
await _reject_realtime_session(
websocket, user_api_key_dict, code=1008, reason="model query parameter is required"
)
return
await _reject_realtime_session(
websocket, user_api_key_dict, code=1008, reason="model query parameter is required"
)
return
assert route_model is not None
try:
await can_key_call_resolved_model(
@ -14064,6 +14158,7 @@ async def realtime_websocket_endpoint(
model=model,
intent=intent,
guardrails=guardrails,
is_translation=is_translation,
)
@ -14077,13 +14172,20 @@ async def _route_realtime_websocket_session(
guardrails: str | None,
request_path: str | None = None,
live_session_start: Mapping[str, object] | None = None,
is_translation: bool = False,
) -> None:
# Only use explicit parameters, not all query params
query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent)))
query_model: Final = route_model if is_translation else model
query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys
RealtimeQueryParams,
dict(_realtime_query_params_template(query_model, intent)),
)
data: dict[str, object] = {
"model": route_model,
"websocket": websocket,
"query_params": query_params, # Only explicit params
"realtime_mode": "translation" if is_translation else "realtime",
}
# Pass guardrails into data so pre-call guardrail processing picks them up
if guardrails:

198
litellm/proxy/realtime_endpoints/endpoints.py
View file

@ -2,19 +2,24 @@
import json
import time
from collections.abc import Mapping
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from typing import TYPE_CHECKING, Final
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi import status as http_status
from pydantic import ConfigDict, TypeAdapter, with_config
from fastapi.responses import PlainTextResponse
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
@ -43,7 +48,26 @@ router: Final = APIRouter()
_REALTIME_TOKEN_VERSION: Final = "realtime_v1"
_DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview"
_DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper"
_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription")
_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation")
@contextmanager
def _invalid_request_body_answers_400() -> Generator[None, None, None]:
try:
yield
except ValidationError as error:
raise ProxyException(message=str(error), type="invalid_request_error", param=None, code=400) from error
_TRANSLATION_SECRET_BODY: Final = {
"requestBody": inline_request_body(
RealtimeClientSecretRequest,
{"model": "gpt-realtime-translate", "session": {"type": "translation", "model": "gpt-realtime-translate"}},
)
}
_TRANSLATION_SDP_BODY: Final = {
"requestBody": {"required": True, "content": {"application/sdp": {"schema": {"type": "string"}}}}
}
@with_config(ConfigDict(strict=True, extra="allow"))
@ -140,19 +164,60 @@ def _set_transcription_model_on_session(
}
def nested_transcription_upstream_model(model: str, team_id: str | None, llm_router: "Router | None") -> str:
deployment_model: Final = (
llm_router.registry().resolve(model, kind="upstream", team_id=team_id) if llm_router is not None else None
)
if deployment_model is None:
return model
try:
return litellm.get_llm_provider(model=deployment_model)[0]
except litellm.BadRequestError:
return deployment_model
async def _authorize_and_bind_nested_transcription_models(
session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization
user_api_key_dict: UserAPIKeyAuth,
llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list
llm_router: "Router | None",
) -> None:
nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data))
for nested_model in nested_models:
await can_key_call_resolved_model(
model=nested_model,
valid_token=user_api_key_dict,
llm_model_list=llm_model_list,
llm_router=llm_router,
)
if nested_models:
_set_transcription_model_on_session(
session=session_data,
model=nested_transcription_upstream_model(nested_models[0], user_api_key_dict.team_id, llm_router),
)
async def _prepare_client_secret_session(
req: RealtimeClientSecretRequest,
user_api_key_dict: UserAPIKeyAuth,
llm_model_list: list | None,
llm_router: "Router | None",
forced_session_type: str | None = None,
) -> tuple[str, dict | None, str]:
session_type: Final = _coerce_realtime_session_type(req.session.type if req.session else None)
session_data: Final[dict | None] = req.session.model_dump(exclude_none=True) if req.session else None
requested_session_type: Final = req.session.type if req.session else None
if forced_session_type is None and requested_session_type == "translation":
raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint")
session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type)
session_data: Final[dict | None] = (
req.session.model_dump(exclude_unset=True) if req.session else ({} if session_type == "translation" else None)
)
if session_data is not None:
session_data["type"] = session_type
session_model: Final = req.session.model if req.session else None
model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL
requested_model: Final = session_model or req.model
default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL
model: str = requested_model or default_model
if session_type != "transcription":
await can_key_call_resolved_model(
model=model,
@ -160,6 +225,15 @@ async def _prepare_client_secret_session(
llm_model_list=llm_model_list,
llm_router=llm_router,
)
if session_data is not None and (requested_model or session_type == "translation"):
session_data["model"] = model
if session_type == "translation":
await _authorize_and_bind_nested_transcription_models(
session_data=session_data,
user_api_key_dict=user_api_key_dict,
llm_model_list=llm_model_list,
llm_router=llm_router,
)
return model, session_data, session_type
transcription_model_candidates: Final = _transcription_model_candidates_from_session(session_data or {})
@ -194,12 +268,13 @@ def _encode_realtime_token_payload(
team_id: str | None,
expires_at: int | None,
session_type: str = "realtime",
caller_auth: Mapping[str, object] | None = None,
) -> str:
"""
Encode metadata with the upstream ephemeral key so /realtime/calls can
route without requiring model as a query param.
"""
payload: Final[dict[str, str | int | None]] = {
payload: Final[Mapping[str, object]] = {
"v": _REALTIME_TOKEN_VERSION,
"ephemeral_key": ephemeral_key,
"model_id": model_id,
@ -207,10 +282,24 @@ def _encode_realtime_token_payload(
"team_id": team_id or "",
"expires_at": expires_at,
"session_type": session_type,
**({} if caller_auth is None else {"caller_auth": caller_auth}),
}
return json.dumps(payload, separators=(",", ":"))
def _caller_auth_for_token(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object]:
return user_api_key_dict.model_dump(mode="json", exclude_none=True, exclude={"api_key", "parent_otel_span"})
def _caller_auth_from_token(
decoded_payload: Mapping[str, object] | None, user_id: str | None, team_id: str | None
) -> UserAPIKeyAuth:
caller_auth: Final = None if decoded_payload is None else as_str_mapping(decoded_payload.get("caller_auth"))
if caller_auth is None:
return UserAPIKeyAuth(user_id=user_id, team_id=team_id)
return UserAPIKeyAuth.model_validate({**caller_auth, "api_key": caller_auth.get("token")})
def _header_value(headers: Mapping[str, str], name: str, default: str) -> str:
return headers.get(name, default)
@ -249,11 +338,24 @@ def _decode_realtime_token_payload(
dependencies=[Depends(user_api_key_auth)],
tags=["realtime"],
)
@router.post(
"/v1/realtime/translations/client_secrets",
openapi_extra=_TRANSLATION_SECRET_BODY,
dependencies=[Depends(user_api_key_auth)],
tags=["realtime"],
)
@router.post(
"/realtime/translations/client_secrets",
openapi_extra=_TRANSLATION_SECRET_BODY,
dependencies=[Depends(user_api_key_auth)],
tags=["realtime"],
)
async def create_realtime_client_secret(
request: Request,
fastapi_response: Response,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> RealtimeClientSecretResponse:
"""Create a short-lived client token for a Realtime conversation, transcription, or translation session"""
from litellm.proxy.proxy_server import (
add_litellm_data_to_request,
general_settings,
@ -269,13 +371,16 @@ async def create_realtime_client_secret(
data: dict = {}
try:
body: Final = await read_request_body(request=request)
req: Final = RealtimeClientSecretRequest(**body)
with _invalid_request_body_answers_400():
req: Final = RealtimeClientSecretRequest.model_validate(body)
is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path
model, session_data, session_type = await _prepare_client_secret_session(
req=req,
user_api_key_dict=user_api_key_dict,
llm_model_list=llm_model_list,
llm_router=llm_router,
forced_session_type="translation" if is_translation_request else None,
)
data = {"model": model}
@ -299,17 +404,20 @@ async def create_realtime_client_secret(
proxy_config=proxy_config,
)
call_type: Final = (
"acreate_realtime_translation_client_secret" if is_translation_request else "acreate_realtime_client_secret"
)
data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_dict,
data=data,
call_type="acreate_realtime_client_secret",
call_type=call_type,
)
verbose_proxy_logger.debug("WebRTC: /v1/realtime/client_secrets (model=%s)", model)
llm_call: Final = await route_request(
data=data,
route_type="acreate_realtime_client_secret",
route_type=call_type,
llm_router=llm_router,
user_model=user_model,
)
@ -366,6 +474,7 @@ async def create_realtime_client_secret(
team_id=getattr(user_api_key_dict, "team_id", None),
expires_at=expires_at if isinstance(expires_at, int) else None,
session_type=session_type,
caller_auth=_caller_auth_for_token(user_api_key_dict) if session_type == "translation" else None,
)
encrypted_token: Final[str] = encrypt_value_helper(token_payload)
upstream_json["value"] = encrypted_token
@ -392,10 +501,27 @@ async def create_realtime_client_secret(
"/openai/v1/realtime/calls",
tags=["realtime"],
)
@router.post(
"/v1/realtime/translations/calls",
status_code=http_status.HTTP_201_CREATED,
response_class=PlainTextResponse,
response_description="WebRTC SDP answer",
openapi_extra=_TRANSLATION_SDP_BODY,
tags=["realtime"],
)
@router.post(
"/realtime/translations/calls",
status_code=http_status.HTTP_201_CREATED,
response_class=PlainTextResponse,
response_description="WebRTC SDP answer",
openapi_extra=_TRANSLATION_SDP_BODY,
tags=["realtime"],
)
async def proxy_realtime_calls(
request: Request,
fastapi_response: Response,
) -> Response:
"""Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets"""
from litellm.proxy.proxy_server import (
add_litellm_data_to_request,
general_settings,
@ -417,6 +543,7 @@ async def proxy_realtime_calls(
media_type="application/json",
)
is_translation_request: Final = "/realtime/translations/calls" in request.url.path
encrypted_token: Final = auth_header.removeprefix("Bearer ").strip()
decrypted_token_value: Final = decrypt_value_helper(
value=encrypted_token,
@ -434,33 +561,39 @@ async def proxy_realtime_calls(
if decoded_payload is not None:
# Check token expiry
expires_at: Final = decoded_payload.get("expires_at")
if expires_at is not None and isinstance(expires_at, int):
if time.time() > expires_at:
return Response(
content=json.dumps({"error": "Token has expired"}),
status_code=http_status.HTTP_401_UNAUTHORIZED,
media_type="application/json",
)
if isinstance(expires_at, int) and time.time() > expires_at:
return Response(
content=json.dumps({"error": "Token has expired"}),
status_code=http_status.HTTP_401_UNAUTHORIZED,
media_type="application/json",
)
openai_ephemeral_key = decoded_payload.get("ephemeral_key", "")
model = decoded_payload.get("model_id") or request.query_params.get("model") or _DEFAULT_REALTIME_MODEL
user_id = decoded_payload.get("user_id") or None
team_id = decoded_payload.get("team_id") or None
session_type = _coerce_realtime_session_type(decoded_payload.get("session_type"))
raw_session_type: Final = decoded_payload.get("session_type")
session_type = _coerce_realtime_session_type(raw_session_type)
if is_translation_request != (raw_session_type == "translation"):
return Response(
content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}),
status_code=http_status.HTTP_401_UNAUTHORIZED,
media_type="application/json",
)
else:
# Backward compatibility: older tokens contained only encrypted upstream key.
if is_translation_request:
return Response(
content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}),
status_code=http_status.HTTP_401_UNAUTHORIZED,
media_type="application/json",
)
openai_ephemeral_key = decrypted_token_value
model = request.query_params.get("model", _DEFAULT_REALTIME_MODEL)
user_id = None
team_id = None
session_type = "realtime"
# Build a minimal UserAPIKeyAuth with user/team IDs from the token
# so spend tracking and budget enforcement work correctly.
minimal_auth: Final = UserAPIKeyAuth(
user_id=user_id,
team_id=team_id,
)
token_caller_auth: Final = _caller_auth_from_token(decoded_payload, user_id, team_id)
data: dict = {}
try:
@ -487,22 +620,23 @@ async def proxy_realtime_calls(
data=data,
request=request,
general_settings=general_settings,
user_api_key_dict=minimal_auth,
user_api_key_dict=token_caller_auth,
version=version,
proxy_config=proxy_config,
)
call_type: Final = "arealtime_translation_calls" if is_translation_request else "arealtime_calls"
data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=minimal_auth,
user_api_key_dict=token_caller_auth,
data=data,
call_type="arealtime_calls",
call_type=call_type,
)
verbose_proxy_logger.debug("WebRTC: /v1/realtime/calls (model=%s)", model)
llm_call: Final = await route_request(
data=data,
route_type="arealtime_calls",
route_type=call_type,
llm_router=llm_router,
user_model=user_model,
)
@ -510,7 +644,7 @@ async def proxy_realtime_calls(
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=minimal_auth,
user_api_key_dict=token_caller_auth,
original_exception=e,
request_data=data,
)
@ -581,7 +715,8 @@ async def create_realtime_transcription_session(
data: dict = {}
try:
body: Final = await read_request_body(request=request)
req: Final = RealtimeTranscriptionSessionRequest(**body)
with _invalid_request_body_answers_400():
req: Final = RealtimeTranscriptionSessionRequest.model_validate(body)
model: Final[str] = req.resolved_model() or "gpt-realtime-whisper"
await can_key_call_resolved_model(
@ -592,6 +727,7 @@ async def create_realtime_transcription_session(
)
transcription_session: Final = {k: v for k, v in body.items() if k != "model"}
_set_transcription_model_on_session(session=transcription_session, model=model, create_if_missing=True)
data = {"model": model, "transcription_session": transcription_session}
data = await add_litellm_data_to_request(

2
litellm/proxy/route_llm_request.py
View file

@ -621,7 +621,9 @@ RouteType = Literal[
"alist_input_items",
"_arealtime", # private function for realtime API
"acreate_realtime_client_secret",
"acreate_realtime_translation_client_secret",
"arealtime_calls",
"arealtime_translation_calls",
"acreate_realtime_transcription_session",
"_aresponses_websocket", # private function for responses WebSocket mode
"aimage_edit",

10
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -598,6 +598,7 @@ def get_logging_payload(
kwargs = {}
rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError)
failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None
response_served_from_cache: Final = served_from_cache(response_obj)
if response_obj is None:
response_obj = {}
@ -888,14 +889,11 @@ def get_logging_payload(
additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens
clean_metadata["additional_usage_values"] = additional_usage_values
drop_requester_ip_address: Final = requester_ip_address_logging_disabled()
error_information: Final = clean_metadata["error_information"]
if drop_requester_ip_address:
clean_metadata["requester_ip_address"] = None
if drop_requester_ip_address and isinstance(error_information, Mapping):
clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked
StandardLoggingPayloadErrorInformation,
_error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)),
)
clean_metadata["error_information"] = sanitize_error_information_for_spend_logs(
clean_metadata["error_information"], original_exception=failure_exception
)
if litellm.cache is None:
cache_key = "Cache OFF"

484
litellm/realtime_api/main.py
View file

@ -2,11 +2,13 @@
import asyncio
import os
import traceback
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast
import httpx
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, TypedDict, assert_never
@ -14,10 +16,14 @@ import litellm
from litellm.constants import (
AZURE_OPENAI_AUDIO_PROVIDERS,
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
request_timeout,
)
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.xai.common_utils import XAIModelInfo
@ -36,18 +42,24 @@ from litellm.utils import (
ModelInfoLookupMiss,
ProviderConfigManager,
aresolve_api_key_secret_reference,
get_base_model_from_metadata,
lookup_model_info_for_registration,
)
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
from ..litellm_core_utils.core_helpers import normalize_drop_params
from ..litellm_core_utils.credential_accessor import CredentialAccessor
from ..litellm_core_utils.get_litellm_params import get_litellm_params
from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from ..litellm_core_utils.realtime_streaming import client_websocket_scope
from ..litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
from ..litellm_core_utils.realtime_streaming import (
TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE,
client_websocket_scope,
translation_guardrails_apply,
)
from ..llms.azure.common_utils import get_azure_ad_token
from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client
from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed
from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
from ..llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, realtime_ssl_for_url
from ..llms.openai.common_utils import is_openai_backed_api_base
from ..llms.openai.live.handler import OpenAILiveSessions
from ..llms.openai.realtime.handler import (
@ -66,8 +78,6 @@ from ..utils import client as wrapper_client
if TYPE_CHECKING:
from fastapi import WebSocket
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
azure_realtime: Final = AzureOpenAIRealtime()
openai_realtime: Final = OpenAIRealtime()
openai_live_sessions: Final = OpenAILiveSessions()
@ -77,7 +87,7 @@ vertex_llm_base: Final = VertexBase()
base_llm_http_handler = BaseLLMHTTPHandler()
_EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
_DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object])
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]:
@ -90,26 +100,145 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) ->
return MappingProxyType({**credential_values, **model_params})
def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]:
if "model" not in session:
return session
return {**session, "model": model_name}
def _transcription_model_overrides(session: Mapping[str, object], model_name: str) -> Mapping[str, object]:
flat_transcription: Final = as_str_mapping(session.get("input_audio_transcription"))
audio: Final = as_str_mapping(session.get("audio")) or {}
audio_input: Final = as_str_mapping(audio.get("input")) or {}
nested_transcription: Final = as_str_mapping(audio_input.get("transcription"))
flat_part: Final = (
{} if flat_transcription is None else {"input_audio_transcription": {**flat_transcription, "model": model_name}}
)
nested_part: Final = (
{}
if nested_transcription is None and flat_transcription is not None
else {
"audio": {
**audio,
"input": {**audio_input, "transcription": {**(nested_transcription or {}), "model": model_name}},
}
}
)
return MappingProxyType({**flat_part, **nested_part})
def _resolved_request_session(request_data: Mapping[str, object], model_name: str) -> Mapping[str, object] | None:
session: Final = as_str_mapping(request_data.get("session"))
if not isinstance(session, dict):
return None
return _with_resolved_session_model(session, model_name)
def with_resolved_session_model(session: Mapping[str, object], model_name: str) -> dict[str, object]:
top_level_model: Final = {} if "model" not in session else {"model": model_name}
transcription_models: Final = (
_transcription_model_overrides(session, model_name) if session.get("type") == "transcription" else {}
)
return {**session, **top_level_model, **transcription_models}
def _build_litellm_metadata(kwargs: dict) -> dict[str, object]:
"""Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider)."""
metadata: Final[dict[str, object]] = {**(kwargs.get("litellm_metadata") or {})}
guardrails: Final = (kwargs.get("metadata") or {}).get("guardrails") or kwargs.get("guardrails") or []
if guardrails:
metadata["guardrails"] = guardrails
return metadata
def _without_top_level_nulls(fields: Mapping[str, object]) -> dict[str, object]:
return {key: value for key, value in fields.items() if value is not None}
_POLICY_PIPELINE_METADATA_KEYS: Final = ("_guardrail_pipelines", "_pipeline_managed_guardrails")
def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]:
request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {})
internal_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("litellm_metadata") or {})
policy_pipeline_state: Final = {
key: request_metadata[key] for key in _POLICY_PIPELINE_METADATA_KEYS if key in request_metadata
}
guardrails: Final = request_metadata.get("guardrails") or kwargs.get("guardrails")
return {**policy_pipeline_state, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})}
_GUARDRAIL_OPT_OUT_METADATA_KEYS: Final = ("user_api_key_metadata", "user_api_key_team_metadata")
def _translation_litellm_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]:
request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {})
guardrail_opt_outs: Final = {
key: request_metadata[key] for key in _GUARDRAIL_OPT_OUT_METADATA_KEYS if key in request_metadata
}
return MappingProxyType({**guardrail_opt_outs, **_build_litellm_metadata(kwargs)})
_HTTP_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value})
_WEBSOCKET_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, *AZURE_OPENAI_AUDIO_PROVIDERS})
@dataclass(frozen=True, slots=True)
class _TranslationProviderUnsupported:
provider: str
supported_providers: frozenset[str]
@dataclass(frozen=True, slots=True)
class _TranslationGuardrailsUnsupported:
pass
_TranslationRejection = _TranslationProviderUnsupported | _TranslationGuardrailsUnsupported
def _translation_rejection(
custom_llm_provider: str, supported_providers: frozenset[str], litellm_metadata: Mapping[str, object]
) -> _TranslationRejection | None:
if custom_llm_provider not in supported_providers:
return _TranslationProviderUnsupported(provider=custom_llm_provider, supported_providers=supported_providers)
if translation_guardrails_apply({"litellm_metadata": dict(litellm_metadata)}):
return _TranslationGuardrailsUnsupported()
return None
def _translation_rejection_message(rejection: _TranslationRejection) -> str:
match rejection:
case _TranslationProviderUnsupported(provider=provider, supported_providers=supported_providers):
return (
f"Realtime translation supports only the {', '.join(sorted(supported_providers))} providers, "
f"got {provider}"
)
case _TranslationGuardrailsUnsupported():
return TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE
case _:
assert_never(rejection)
def _translation_rejection_error_type(rejection: _TranslationRejection) -> str:
match rejection:
case _TranslationProviderUnsupported():
return "invalid_request_error"
case _TranslationGuardrailsUnsupported():
return "guardrail_error"
case _:
assert_never(rejection)
def _translation_rejection_error(
rejection: _TranslationRejection, model: str, custom_llm_provider: str
) -> BadRequestError:
return BadRequestError(
message=_translation_rejection_message(rejection), model=model, llm_provider=custom_llm_provider
)
def _raise_if_http_translation_rejected(custom_llm_provider: str, model: str, kwargs: Mapping[str, object]) -> None:
rejection: Final = _translation_rejection(
custom_llm_provider, _HTTP_TRANSLATION_PROVIDERS, _translation_litellm_metadata(kwargs)
)
if rejection is None:
return
raise _translation_rejection_error(rejection, model, custom_llm_provider)
async def _refuse_translation_session(
websocket: "WebSocket",
rejection: _TranslationRejection,
logging_obj: LiteLLMLogging,
model: str,
custom_llm_provider: str,
) -> None:
message: Final = _translation_rejection_message(rejection)
await websocket.send_text(realtime_error_event(message, error_type=_translation_rejection_error_type(rejection)))
await websocket.close(code=1008, reason=websocket_close_reason(message, fallback="Translation session refused"))
error: Final = _translation_rejection_error(rejection, model, custom_llm_provider)
await logging_obj.dispatch_failure_handlers(
error, "".join(traceback.format_exception(error)), prefer_async_handlers=True
)
def _get_realtime_http_provider_config(
@ -125,10 +254,6 @@ def _get_realtime_http_provider_config(
Uses ProviderConfigManager so each provider keeps its credential-resolution
and URL-construction logic in its own transformation class.
"""
from litellm.llms.base_llm.realtime.http_transformation import (
BaseRealtimeHTTPConfig,
)
provider_config: BaseRealtimeHTTPConfig | None = None
if custom_llm_provider in LlmProviders._member_map_.values():
provider_config = ProviderConfigManager.get_provider_realtime_http_config(
@ -163,6 +288,32 @@ async def _realtime_http_bearer_token(
return bearer_token or ""
async def _get_realtime_http_extra_headers(
provider_config: BaseRealtimeHTTPConfig | None,
litellm_params: GenericLiteLLMParams,
resolved_api_key: str,
extra_headers: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
if provider_config is None:
return extra_headers
return await asyncio.to_thread(
provider_config.get_extra_headers,
litellm_params=litellm_params,
api_key=resolved_api_key,
extra_headers=extra_headers,
)
def _client_secret_session_body(req: RealtimeClientSecretRequest, model_name: str) -> Mapping[str, object] | None:
if req.session is not None:
return with_resolved_session_model(
_without_top_level_nulls(req.session.model_dump(exclude_unset=True)), model_name
)
if req.model is not None:
return {"type": "realtime", "model": model_name}
return None
@wrapper_client
async def acreate_realtime_client_secret(
model: str | None = None,
@ -176,31 +327,42 @@ async def acreate_realtime_client_secret(
session=RealtimeSessionConfig.model_validate(session) if session else None,
expires_after=RealtimeExpiresAfter.model_validate(expires_after) if expires_after else None,
)
model_name = (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview"
transcription_model: Final = (
req.session.audio.input.transcription.model
if req.session is not None
and req.session.type == "transcription"
and req.session.audio is not None
and req.session.audio.input is not None
and req.session.audio.input.transcription is not None
else None
)
requested_model_name: Final = (
transcription_model
or (req.session.model if req.session is not None else None)
or req.model
or "gpt-4o-realtime-preview"
)
litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj")
litellm_params: Final = GenericLiteLLMParams(**kwargs)
(
model_name,
custom_llm_provider,
dynamic_api_key,
dynamic_api_base,
) = get_llm_provider(
model=model_name,
model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider(
model=requested_model_name,
custom_llm_provider=litellm_params.custom_llm_provider,
api_base=litellm_params.api_base,
api_key=litellm_params.api_key,
)
(
provider_config,
resolved_api_base,
resolved_api_key,
) = _get_realtime_http_provider_config(
provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config(
custom_llm_provider=custom_llm_provider,
dynamic_api_base=dynamic_api_base,
dynamic_api_key=dynamic_api_key,
litellm_params=litellm_params,
)
resolved_extra_headers: Final = await _get_realtime_http_extra_headers(
provider_config=provider_config,
litellm_params=litellm_params,
resolved_api_key=resolved_api_key,
extra_headers=kwargs.get("extra_headers"),
)
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model_name,
@ -208,26 +370,124 @@ async def acreate_realtime_client_secret(
litellm_params={"api_base": resolved_api_base},
custom_llm_provider=custom_llm_provider,
)
request_data: Final = req.model_dump(exclude_none=True, exclude={"model"})
resolved_session: Final = _resolved_request_session(request_data, model_name)
if resolved_session is not None:
request_data["session"] = resolved_session
session_body: Final = _client_secret_session_body(req, model_name)
return await base_llm_http_handler.async_realtime_client_secret_handler(
api_base=resolved_api_base,
api_key=await _realtime_http_bearer_token(
custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
),
request_data=request_data,
request_data={
**req.model_dump(exclude_none=True, exclude={"model", "session"}),
**({} if session_body is None else {"session": session_body}),
},
logging_obj=litellm_logging_obj,
timeout=timeout or request_timeout,
provider_config=provider_config,
model=model_name,
extra_headers=kwargs.get("extra_headers"),
extra_headers=resolved_extra_headers,
client=kwargs.get("client"),
api_version=litellm_params.api_version,
)
@wrapper_client
async def acreate_realtime_translation_client_secret(
model: str | None = None,
session: Mapping[str, Any] | None = None,
expires_after: Mapping[str, Any] | None = None,
timeout: float | None = None,
**kwargs: object, # kwargs-ok: public API preserves provider-specific request options
) -> httpx.Response:
requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate"
session_config: Final = RealtimeSessionConfig.model_validate(
{
**(session or {}),
"type": "translation",
"model": requested_model_name,
}
)
req: Final = RealtimeClientSecretRequest(
model=requested_model_name,
session=session_config,
expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None,
)
litellm_logging_obj: Final = kwargs.get("litellm_logging_obj")
if not isinstance(litellm_logging_obj, LiteLLMLogging):
raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance")
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)
model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider(
model=requested_model_name,
custom_llm_provider=litellm_params.custom_llm_provider,
api_base=litellm_params.api_base,
api_key=litellm_params.api_key,
)
_raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs)
provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config(
custom_llm_provider=custom_llm_provider,
dynamic_api_base=dynamic_api_base,
dynamic_api_key=dynamic_api_key,
litellm_params=litellm_params,
)
resolved_extra_headers: Final = await _get_realtime_http_extra_headers(
provider_config=provider_config,
litellm_params=litellm_params,
resolved_api_key=resolved_api_key,
extra_headers=as_str_mapping(kwargs.get("extra_headers")),
)
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model_name,
optional_params={
"expires_after": expires_after,
"session": session,
},
litellm_params={"api_base": resolved_api_base},
custom_llm_provider=custom_llm_provider,
)
translation_session: Final = with_resolved_session_model(
_without_top_level_nulls(session_config.model_dump(exclude_unset=True)), model_name
)
return await base_llm_http_handler.async_realtime_translation_client_secret_handler(
api_base=resolved_api_base,
api_key=await _realtime_http_bearer_token(
custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
),
request_data={
**req.model_dump(exclude_none=True, exclude={"model", "session"}),
"session": {key: value for key, value in translation_session.items() if key != "type"},
},
logging_obj=litellm_logging_obj,
timeout=timeout or request_timeout,
provider_config=provider_config,
model=model_name,
extra_headers=resolved_extra_headers,
client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None,
api_version=litellm_params.api_version,
)
def _with_upstream_transcription_model(request_data: Mapping[str, object], model_name: str) -> dict[str, object]:
flat: Final = as_str_mapping(request_data.get("input_audio_transcription"))
audio: Final = as_str_mapping(request_data.get("audio")) or {}
audio_input: Final = as_str_mapping(audio.get("input")) or {}
nested: Final = as_str_mapping(audio_input.get("transcription"))
return {
**request_data,
**({} if flat is None else {"input_audio_transcription": {**flat, "model": model_name}}),
**(
{}
if nested is None
else {"audio": {**audio, "input": {**audio_input, "transcription": {**nested, "model": model_name}}}}
),
}
def _deployment_base_model(kwargs: Mapping[str, object], custom_llm_provider: str) -> str | None:
base_model: Final = get_base_model_from_metadata(model_call_details={"litellm_params": kwargs})
return None if base_model is None else base_model.removeprefix(f"{custom_llm_provider}/")
@wrapper_client
async def acreate_realtime_transcription_session(
model: str | None = None,
@ -273,6 +533,12 @@ async def acreate_realtime_transcription_session(
dynamic_api_key=dynamic_api_key,
litellm_params=litellm_params,
)
resolved_extra_headers: Final = await _get_realtime_http_extra_headers(
provider_config=provider_config,
litellm_params=litellm_params,
resolved_api_key=resolved_api_key,
extra_headers=kwargs.get("extra_headers"),
)
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model_name,
@ -280,13 +546,11 @@ async def acreate_realtime_transcription_session(
litellm_params={"api_base": resolved_api_base},
custom_llm_provider=custom_llm_provider,
)
request_data: Final = req.model_dump(exclude_none=True, exclude={"model"})
# Ensure the upstream body's input_audio_transcription.model matches the
# authorized routing model. This prevents a caller from supplying an allowed
# top-level model for auth while sneaking a different model into the nested
# transcription config that gets forwarded to the provider.
if isinstance(request_data.get("input_audio_transcription"), dict):
request_data["input_audio_transcription"]["model"] = model_name
# The upstream transcription model must match the authorized routing model, so a
# caller cannot pass auth with an allowed top-level model and forward a different one.
request_data: Final = _with_upstream_transcription_model(
_without_top_level_nulls(req.model_dump(exclude_unset=True, exclude={"model"})), model_name
)
return await base_llm_http_handler.async_realtime_transcription_session_handler(
api_base=resolved_api_base,
api_key=await _realtime_http_bearer_token(
@ -297,7 +561,7 @@ async def acreate_realtime_transcription_session(
timeout=timeout or request_timeout,
provider_config=provider_config,
model=model_name,
extra_headers=kwargs.get("extra_headers"),
extra_headers=resolved_extra_headers,
client=kwargs.get("client"),
api_version=litellm_params.api_version,
)
@ -334,7 +598,7 @@ async def arealtime_calls(
litellm_params=litellm_params,
)
if session is not None:
session = _with_resolved_session_model(session, model_name)
session = with_resolved_session_model(session, model_name)
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model_name,
@ -357,6 +621,68 @@ async def arealtime_calls(
)
@wrapper_client
async def arealtime_translation_calls(
openai_ephemeral_key: str,
sdp_body: bytes,
model: str | None = None,
session: Mapping[str, Any] | None = None,
timeout: float | None = None,
**kwargs: object, # kwargs-ok: public API preserves provider-specific request options
) -> httpx.Response:
requested_model_name: Final = model or "gpt-realtime-translate"
litellm_logging_obj: Final = kwargs.get("litellm_logging_obj")
if not isinstance(litellm_logging_obj, LiteLLMLogging):
raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance")
litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs)
model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider(
model=requested_model_name,
custom_llm_provider=litellm_params.custom_llm_provider,
api_base=litellm_params.api_base,
api_key=litellm_params.api_key,
)
_raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs)
provider_config, resolved_api_base, _ = _get_realtime_http_provider_config(
custom_llm_provider=custom_llm_provider,
dynamic_api_base=dynamic_api_base,
dynamic_api_key=dynamic_api_key,
litellm_params=litellm_params,
)
session_config: Final = with_resolved_session_model(
{
**(session or {}),
"type": "translation",
"model": model_name,
},
model_name,
)
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model_name,
optional_params={
"realtime_translation_calls": True,
"session": session_config,
},
litellm_params={"api_base": resolved_api_base},
custom_llm_provider=custom_llm_provider,
)
return await base_llm_http_handler.async_realtime_calls_handler(
api_base=resolved_api_base,
openai_ephemeral_key=openai_ephemeral_key,
sdp_body=sdp_body,
logging_obj=litellm_logging_obj,
timeout=timeout or request_timeout,
provider_config=provider_config,
model=model_name,
session_config=session_config,
extra_headers=as_str_mapping(kwargs.get("extra_headers")),
client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None,
api_version=litellm_params.api_version,
translation=True,
)
async def vertex_access_token_resolver(
credentials: VERTEX_CREDENTIALS_TYPES | None,
project_id: str | None,
@ -486,6 +812,7 @@ async def _arealtime(
timeout: float | None = None,
query_params: RealtimeQueryParams | None = None,
live_session_start: Mapping[str, object] | None = None,
realtime_mode: str = "realtime",
**kwargs,
) -> None:
"""
@ -530,7 +857,24 @@ async def _arealtime(
custom_llm_provider=_custom_llm_provider,
)
deployment_params: Final = _DEPLOYMENT_PARAMS.validate_python(kwargs)
session_litellm_metadata: Final = (
dict(_translation_litellm_metadata(kwargs))
if realtime_mode == "translation"
else _build_litellm_metadata(kwargs)
)
translation_rejection: Final = (
_translation_rejection(_custom_llm_provider, _WEBSOCKET_TRANSLATION_PROVIDERS, session_litellm_metadata)
if realtime_mode == "translation"
else None
)
if translation_rejection is not None:
await _refuse_translation_session(
websocket, translation_rejection, litellm_logging_obj, model, _custom_llm_provider
)
litellm_logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] = True
return
deployment_params: Final = _OBJECT_MAPPING.validate_python(kwargs)
if live_session_start is not None:
await _alive_session(
model=model,
@ -540,7 +884,7 @@ async def _arealtime(
session_start=live_session_start,
api_base=dynamic_api_base or litellm_params.api_base,
api_key=dynamic_api_key,
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_metadata=session_litellm_metadata,
litellm_params=deployment_params,
)
return
@ -566,7 +910,7 @@ async def _arealtime(
timeout=timeout,
headers=headers,
user_api_key_dict=kwargs.get("user_api_key_dict"),
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_metadata=session_litellm_metadata,
query_params=query_params,
)
elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
@ -581,9 +925,6 @@ async def _arealtime(
or litellm_params.get("realtime_protocol")
or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL")
)
realtime_protocol: Final = azure_realtime_protocol_for_client(
configured_realtime_protocol, query_params=query_params, websocket=websocket
)
resolved_azure_ad_token: Final = (
None
if api_key
@ -592,6 +933,14 @@ async def _arealtime(
GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token),
)
)
realtime_protocol: Final = azure_realtime_protocol_for_client(
configured_realtime_protocol,
model=model,
base_model=_deployment_base_model(kwargs, _custom_llm_provider),
query_params=query_params,
realtime_mode=realtime_mode,
websocket=websocket,
)
await azure_realtime.async_realtime(
model=model,
websocket=websocket,
@ -602,10 +951,13 @@ async def _arealtime(
client=None,
timeout=timeout,
logging_obj=litellm_logging_obj,
realtime_protocol=realtime_protocol,
realtime_protocol=realtime_protocol.name,
query_params=query_params,
realtime_mode=realtime_mode,
user_api_key_dict=kwargs.get("user_api_key_dict"),
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_metadata=session_litellm_metadata,
bridges_beta_client_to_ga=realtime_protocol.bridges_beta_client_to_ga,
drop_params=drop_params,
)
elif _custom_llm_provider == "openai":
api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/"
@ -622,8 +974,10 @@ async def _arealtime(
client=None,
timeout=timeout,
query_params=query_params,
realtime_mode=realtime_mode,
user_api_key_dict=kwargs.get("user_api_key_dict"),
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_metadata=session_litellm_metadata,
drop_params=drop_params,
)
elif _custom_llm_provider == "bedrock":
# Extract AWS parameters from kwargs
@ -678,7 +1032,7 @@ async def _arealtime(
timeout=timeout,
query_params=query_params,
user_api_key_dict=kwargs.get("user_api_key_dict"),
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_metadata=session_litellm_metadata,
)
elif _custom_llm_provider == "vertex_ai":
vertex_credentials: Final = (
@ -738,7 +1092,7 @@ async def _arealtime(
timeout=timeout,
headers=headers,
user_api_key_dict=kwargs.get("user_api_key_dict"),
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_metadata=session_litellm_metadata,
query_params=query_params,
)
else:

23
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -52,6 +52,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
cache_control_count,
dropped_cache_control,
newest_marks_within_cache_control_limit,
routed_model_info,
whole_messages_within_cache_control_limit,
with_cache_control,
)
@ -782,13 +783,15 @@ def _replays_compaction_block(input: str | ResponseInputParam, context_managemen
_PROVIDERS_KEEPING_MESSAGE_CACHE_CONTROL: Final = frozenset(("anthropic", "dashscope", "minimax", "sap", "zai"))
def _bedrock_target_keeps_item_marks(model: str, drop_params: bool) -> bool:
declared: Final = bedrock_declared_cache_point_support(model)
def _bedrock_target_keeps_item_marks(model: str, drop_params: bool, model_info: Mapping[str, object] | None) -> bool:
declared: Final = bedrock_declared_cache_point_support(model, model_info)
return declared is True or (declared is None and not drop_params)
def _target_keeps_item_marks(model: str, custom_llm_provider: str | None, drop_params: bool) -> bool:
if supports_anthropic_cache_control(model, custom_llm_provider):
def _target_keeps_item_marks(
model: str, custom_llm_provider: str | None, drop_params: bool, model_info: Mapping[str, object] | None
) -> bool:
if supports_anthropic_cache_control(model, custom_llm_provider, model_info):
return True
if custom_llm_provider in _PROVIDERS_KEEPING_MESSAGE_CACHE_CONTROL:
return True
@ -798,7 +801,7 @@ def _target_keeps_item_marks(model: str, custom_llm_provider: str | None, drop_p
return is_snowflake_claude_model(model)
if custom_llm_provider == "databricks":
return "claude" in model.lower()
return custom_llm_provider == "bedrock" and _bedrock_target_keeps_item_marks(model, drop_params)
return custom_llm_provider == "bedrock" and _bedrock_target_keeps_item_marks(model, drop_params, model_info)
def _refuses_marks_on_tool_blocks(model: str, custom_llm_provider: str | None) -> bool:
@ -1234,7 +1237,10 @@ class LiteLLMCompletionResponsesConfig:
request_kwargs: Final = _STR_KEY_DICT_ADAPTER.validate_python({**kwargs, "drop_params": drop_params})
context_management: Final = responses_api_request.get("context_management")
keeps_item_marks: Final = _target_keeps_item_marks(
model, custom_llm_provider, drops_unsupported_params(request_kwargs.get("drop_params"))
model,
custom_llm_provider,
drops_unsupported_params(request_kwargs.get("drop_params")),
routed_model_info(request_kwargs.get("model_info")),
)
request_input: Final = with_compaction_items_as_params(input)
native_compaction: Final = _replays_compaction_block(
@ -1456,7 +1462,10 @@ class LiteLLMCompletionResponsesConfig:
model: Final = str(request_fields.get("model") or "")
provider: Final = requested_provider if isinstance(requested_provider, str) else None
keeps_item_marks: Final = _target_keeps_item_marks(
model, provider, drops_unsupported_params(request_kwargs.get("drop_params"))
model,
provider,
drops_unsupported_params(request_kwargs.get("drop_params")),
routed_model_info(request_kwargs.get("model_info")),
)
lookup: Final = session_lookup or partial(
ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id,

2
litellm/responses/main.py
View file

@ -27,6 +27,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
CARRY_UNMATCHED_MESSAGE_POINTS,
AnthropicCacheControlHook,
configured_injection_points,
routed_model_info,
)
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
@ -882,6 +883,7 @@ def _stamp_injection_points_for_the_hook(
custom_llm_provider,
kwargs.get("api_base") or kwargs.get("base_url"),
kwargs.get("prompt_cache_options"),
routed_model_info(kwargs.get("model_info")),
)

37
litellm/router_python_core.py
View file

@ -84,6 +84,8 @@ from litellm.constants import (
MID_STREAM_FAILURE_LOG_WAIT_SECONDS,
OUTPUT_TOKEN_CEILING_PARAMS,
PROMPT_CACHE_PIN_CACHE_MAX_SIZE,
REALTIME_SESSION_ENDED_ABNORMALLY_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
ROUTING_REQUEST_TAGS_METADATA_KEY,
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
@ -672,6 +674,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType
_NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_CALL_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
@ -847,9 +850,11 @@ def _with_router_resolved_session_model(session: object, model_name: str) -> Map
typed_session: Final = _SESSION_ADAPTER.validate_python(session)
except ValidationError:
return _NO_SESSION_KWARGS
if "model" not in typed_session:
if "model" not in typed_session and typed_session.get("type") != "transcription":
return _NO_SESSION_KWARGS
return MappingProxyType({"session": {**typed_session, "model": model_name}})
from litellm.realtime_api.main import with_resolved_session_model
return MappingProxyType({"session": with_resolved_session_model(dict(typed_session), model_name)})
# Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks
@ -2865,6 +2870,14 @@ class PythonCore(DeprecatedRouterMembers):
self._host.invoke("acreate_realtime_transcription_session"),
call_type="acreate_realtime_transcription_session",
)
self.acreate_realtime_translation_client_secret = self.factory_function(
self._host.invoke("acreate_realtime_translation_client_secret"),
call_type="acreate_realtime_translation_client_secret",
)
self.arealtime_translation_calls = self.factory_function(
self._host.invoke("arealtime_translation_calls"),
call_type="arealtime_translation_calls",
)
self._aresponses_websocket = self.factory_function(
self._host.invoke("_aresponses_websocket"), call_type="_aresponses_websocket"
)
@ -6892,6 +6905,22 @@ class PythonCore(DeprecatedRouterMembers):
raise safeguard_refusal_error(model=model, stop_details=refusal_details)
self.success_calls[model_name] += 1
realtime_session_logging: Final = (
_CALL_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj")
if generic_function_name == "_arealtime"
else None
)
realtime_session_deployment_id: Final = (
_MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id")
if isinstance(realtime_session_logging, LiteLLMLogging)
and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True
and realtime_session_logging.model_call_details.get(REALTIME_SESSION_ENDED_ABNORMALLY_KEY) is not True
else None
)
if realtime_session_deployment_id is not None:
increment_deployment_successes_for_current_minute(
ctx=self._core_ctx, deployment_id=str(realtime_session_deployment_id)
)
verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
return response
@ -8433,6 +8462,8 @@ class PythonCore(DeprecatedRouterMembers):
"acreate_realtime_client_secret",
"arealtime_calls",
"acreate_realtime_transcription_session",
"acreate_realtime_translation_client_secret",
"arealtime_translation_calls",
"_aresponses_websocket",
"acreate_fine_tuning_job",
"acancel_fine_tuning_job",
@ -8703,6 +8734,8 @@ class PythonCore(DeprecatedRouterMembers):
"acreate_realtime_client_secret",
"arealtime_calls",
"acreate_realtime_transcription_session",
"acreate_realtime_translation_client_secret",
"arealtime_translation_calls",
):
return await self._ageneric_api_call_with_fallbacks(
original_function=original_function,

25
litellm/router_utils/prompt_caching_cache.py
View file

@ -55,6 +55,17 @@ def _sorted_pairs(pairs: Iterable[tuple[str, JsonValue]]) -> tuple[tuple[str, Js
return tuple(sorted(pairs, key=lambda pair: pair[0]))
_CACHE_MARKER_KEYS: Final = frozenset(("cache_control", "prompt_cache_breakpoint"))
def _is_cache_marked(block: Mapping[str, object]) -> bool:
match block:
case {"cache_control": {"type": "ephemeral"}}:
return True
case _:
return block.get("prompt_cache_breakpoint") is not None
def _canonical_bytes(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
@ -66,7 +77,7 @@ def _block_unit(
return _canonical_bytes((envelope, block)), message_run_type
block_type: Final = block.get("type")
block_run_type: Final = block_type if isinstance(block_type, str) and block_type in _TOOL_RUN_BLOCK_TYPES else None
stripped: Final = _sorted_pairs(item for item in block.items() if item[0] != "cache_control")
stripped: Final = _sorted_pairs(item for item in block.items() if item[0] not in _CACHE_MARKER_KEYS)
return _canonical_bytes((envelope, stripped)), message_run_type or block_run_type
@ -264,15 +275,9 @@ class PromptCachingCache:
continue
for content_idx, content_block in enumerate(content):
if isinstance(content_block, dict):
cache_control = content_block.get("cache_control")
if (
cache_control is not None
and isinstance(cache_control, dict)
and cache_control.get("type") == "ephemeral"
):
last_cacheable_message_idx = msg_idx
last_cacheable_content_idx = content_idx
if isinstance(content_block, dict) and _is_cache_marked(content_block):
last_cacheable_message_idx = msg_idx
last_cacheable_content_idx = content_idx
# If no cacheable block found, return empty list (no cacheable prefix)
if last_cacheable_message_idx is None:

2
litellm/secret_managers/aws_secret_manager.py
View file

@ -70,7 +70,7 @@ class AWSKeyManagementService_V2:
is_litellm_license_in_env = True
if is_litellm_license_in_env is False:
raise ValueError(
"AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment."
"AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment."
)
def load_aws_kms(self, use_aws_kms: bool | None):

15
litellm/types/llms/openai.py
View file

@ -1247,9 +1247,12 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]]
OpenAIAudioTranscriptionOptionalParams = Literal[
"language",
"languages",
"keywords",
"prompt",
"temperature",
"response_format",
"stream",
"timestamp_granularities",
"include",
]
@ -2545,6 +2548,17 @@ class OpenAIRealtimeResponseUsage(TypedDict):
output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]]
class OpenAIRealtimeTranslationDurationUsage(TypedDict):
type: ReadOnly[Literal["duration"]]
output_seconds: NotRequired[ReadOnly[float]]
input_seconds: NotRequired[ReadOnly[float]]
class OpenAIRealtimeTranslationClosedEvent(TypedDict):
type: ReadOnly[Literal["session.closed"]]
usage: ReadOnly[OpenAIRealtimeTranslationDurationUsage]
class OpenAIRealtimeEventTypes(Enum):
SESSION_CREATED = "session.created"
# Beta delta event names
@ -2588,6 +2602,7 @@ OpenAIRealtimeEvents = (
| OpenAIRealtimeInputAudioTranscriptionCompleted
| OpenAIRealtimeTranscriptionSessionCreated
| OpenAIRealtimeErrorEvent
| OpenAIRealtimeTranslationClosedEvent
)
OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents]

47
litellm/types/realtime.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping, Sequence
from typing import Any, Literal
from pydantic import ConfigDict
@ -66,6 +67,41 @@ class RealtimeExpiresAfter(LiteLLMBaseModel):
seconds: int | None = None
class RealtimeAudioTranscriptionConfig(LiteLLMBaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
model: str | None = None
delay: str | None = None
keywords: Sequence[str] | None = None
language: str | None = None
languages: Sequence[str] | None = None
prompt: str | None = None
class RealtimeAudioInputConfig(LiteLLMBaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
format: str | Mapping[str, object] | None = None
noise_reduction: Mapping[str, object] | None = None
transcription: RealtimeAudioTranscriptionConfig | None = None
turn_detection: Mapping[str, object] | None = None
class RealtimeAudioOutputConfig(LiteLLMBaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
format: str | Mapping[str, object] | None = None
language: str | None = None
voice: str | Mapping[str, object] | None = None
class RealtimeSessionAudioConfig(LiteLLMBaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
input: RealtimeAudioInputConfig | None = None
output: RealtimeAudioOutputConfig | None = None
class RealtimeSessionConfig(LiteLLMBaseModel):
"""
Session configuration nested inside the client_secrets request body.
@ -75,12 +111,12 @@ class RealtimeSessionConfig(LiteLLMBaseModel):
Extra/unknown fields are passed through unchanged.
"""
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra="allow", frozen=True)
type: str | None = None
model: str | None = None
instructions: str | None = None
audio: dict[str, object] | None = None
audio: RealtimeSessionAudioConfig | None = None
include: list[str] | None = None
max_output_tokens: int | str | None = None
output_modalities: list[str] | None = None
@ -129,17 +165,20 @@ class RealtimeTranscriptionSessionRequest(LiteLLMBaseModel):
unchanged to the provider.
"""
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra="allow", frozen=True)
# LiteLLM-only routing hint — stripped before forwarding upstream.
model: str | None = None
input_audio_transcription: dict[str, Any] | None = None
audio: RealtimeSessionAudioConfig | None = None
def resolved_model(self) -> str | None:
if self.model:
return self.model
if self.input_audio_transcription:
return self.input_audio_transcription.get("model")
if self.audio and self.audio.input and self.audio.input.transcription:
return self.audio.input.transcription.model
return None
@ -151,7 +190,7 @@ class RealtimeTranscriptionSessionResponse(LiteLLMBaseModel):
ephemeral key. Unknown fields pass through unchanged.
"""
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra="allow", frozen=True)
client_secret: dict[str, object] | None = None

53
litellm/types/utils.py
View file

@ -554,6 +554,11 @@ class CallTypes(str, Enum):
aextraction = "aextraction"
arealtime = "_arealtime"
aresponses_websocket = "_aresponses_websocket"
acreate_realtime_client_secret = "acreate_realtime_client_secret"
arealtime_calls = "arealtime_calls"
acreate_realtime_transcription_session = "acreate_realtime_transcription_session"
acreate_realtime_translation_client_secret = "acreate_realtime_translation_client_secret"
arealtime_translation_calls = "arealtime_translation_calls"
create_batch = "create_batch"
acreate_batch = "acreate_batch"
aretrieve_batch = "aretrieve_batch"
@ -794,6 +799,8 @@ CallTypesLiteral = Literal[
"acreate_realtime_client_secret",
"arealtime_calls",
"acreate_realtime_transcription_session",
"acreate_realtime_translation_client_secret",
"arealtime_translation_calls",
]
MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset(
@ -802,6 +809,10 @@ MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset(
# Mapping of API routes to their corresponding call types
API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
"/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,),
"/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,),
"/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,),
"/realtime/translations/calls": (CallTypes.arealtime_translation_calls,),
# Chat Completions
"/chat/completions": [CallTypes.acompletion, CallTypes.completion],
"/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion],
@ -1120,9 +1131,11 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = {
"/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
"/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,),
# Realtime API
"/realtime": [CallTypes.arealtime],
"/v1/realtime": [CallTypes.arealtime],
"/openai/v1/realtime": [CallTypes.arealtime],
"/realtime": (CallTypes.arealtime,),
"/v1/realtime": (CallTypes.arealtime,),
"/openai/v1/realtime": (CallTypes.arealtime,),
"/realtime/translations": (CallTypes.arealtime,),
"/v1/realtime/translations": (CallTypes.arealtime,),
"/live/sessions": (CallTypes.arealtime,),
"/v1/live/sessions": (CallTypes.arealtime,),
"/openai/v1/live/sessions": (CallTypes.arealtime,),
@ -2951,9 +2964,19 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel):
input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None
class TranscriptionDetectedLanguage(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
code: str
_DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, ...])
class TranscriptionResponse(OpenAIObject):
text: str | None = None
usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None
languages: Sequence[TranscriptionDetectedLanguage] | None = None
_hidden_params: dict = {}
_response_headers: dict | None = None
@ -2966,8 +2989,28 @@ class TranscriptionResponse(OpenAIObject):
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
self._hidden_params = hidden_params
def __init__(self, text=None) -> None:
super().__init__(text=text)
def __init__(
self,
text: str | None = None,
usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | Mapping[str, object] | None = None,
languages: Sequence[TranscriptionDetectedLanguage | Mapping[str, object]] | None = None,
**kwargs: object, # kwargs-ok: OpenAI-compatible responses preserve provider extension fields
) -> None:
super().__init__(
text=text, usage=usage, languages=TranscriptionResponse.detected_languages(languages), **kwargs
)
if self.languages is None:
del self.languages
@staticmethod
def detected_languages(value: object) -> tuple[TranscriptionDetectedLanguage, ...] | None:
try:
return _DETECTED_LANGUAGES.validate_python(value, from_attributes=True)
except ValidationError:
return None
def set_audio_transcription_duration(self, duration: float) -> None:
self._hidden_params["audio_transcription_duration"] = duration
def __contains__(self, key) -> bool:
# Define custom behavior for the 'in' operator

232
litellm/utils.py
View file

@ -1124,6 +1124,38 @@ def _is_streaming_response_for_correlation(result: object) -> bool:
return isinstance(result, CustomStreamWrapper)
def _returns_requested_stream(kwargs: dict[str, object], call_type: CallTypes | str, result: object) -> bool:
if call_type in (CallTypes.transcription, CallTypes.atranscription):
return isinstance(result, (openai.Stream, openai.AsyncStream))
return _is_streaming_request(kwargs=kwargs, call_type=call_type)
def _transcription_runs_unstreamed(
call_type: str, requested_kwargs: Mapping[str, object], call_kwargs: Mapping[str, object]
) -> bool:
from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested
return (
call_type == CallTypes.atranscription.value
and transcription_stream_requested(requested_kwargs.get("stream"))
and call_kwargs.get("stream") is False
)
def _guardrail_view_of_requested_stream(call_kwargs: Mapping[str, object]) -> dict[str, object]:
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming
return guardrail_request_data_with_streaming(call_kwargs, is_streaming=True)
def _replayed_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object:
from litellm.litellm_core_utils.audio_utils.transcription_streaming import GuardedTranscriptionReplay
if not runs_unstreamed or not isinstance(result, TranscriptionResponse):
return result
return GuardedTranscriptionReplay(result)
def _is_converted_stream_result(result: object) -> bool:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
@ -1559,7 +1591,6 @@ def function_setup(
applied_guardrails=applied_guardrails,
supports_correlation_logging=is_async_call,
)
## check if metadata is passed in
litellm_params: Final[dict[str, object]] = {"api_base": ""}
if "metadata" in kwargs:
@ -1807,7 +1838,28 @@ async def async_pre_call_deployment_hook(kwargs: Mapping[str, object], call_type
if result is not None:
modified_kwargs = result
return modified_kwargs
return _unstreamed_when_post_call_guardrails_apply(modified_kwargs, typed_call_type)
def _unstreamed_when_post_call_guardrails_apply(
kwargs: dict[str, object], call_type: CallTypes | None
) -> dict[str, object]:
from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested
if (
call_type is not CallTypes.atranscription
or not transcription_stream_requested(kwargs.get("stream"))
or _deployment_provider(kwargs) not in (LlmProviders.OPENAI.value, LlmProviders.AZURE.value)
):
return kwargs
streamed_request: Final = _guardrail_view_of_requested_stream(kwargs)
custom_guardrail_class: Final = _get_cached_custom_guardrail()
if not any(
isinstance(callback, custom_guardrail_class) and callback.runs_post_call_deployment_hook(streamed_request)
for callback in litellm.callbacks
):
return kwargs
return {**kwargs, "stream": False}
async def async_post_call_success_deployment_hook(
@ -2155,7 +2207,7 @@ def client(original_function):
# MODEL CALL
result = original_function(*args, **kwargs)
end_time = datetime.datetime.now()
if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result):
if _returns_requested_stream(kwargs, call_type, result) or _is_converted_stream_result(result):
if "complete_response" in kwargs and kwargs["complete_response"] is True:
chunks = []
for idx, chunk in enumerate(result):
@ -2164,6 +2216,15 @@ def client(original_function):
else:
logging_obj.stream = True
logging_obj.model_call_details["stream"] = True
if call_type == CallTypes.transcription.value and isinstance(result, openai.Stream):
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
litellm_exception_mapper,
wrap_transcription_stream,
)
result = wrap_transcription_stream(
result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details)
)
# RETURN RESULT
update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata
update_response_metadata(
@ -2339,6 +2400,7 @@ def client(original_function):
kwargs["litellm_logging_obj"] = logging_obj
modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type)
transcription_runs_unstreamed: Final = _transcription_runs_unstreamed(call_type, kwargs, modified_kwargs)
if modified_kwargs is not None:
kwargs = modified_kwargs
@ -2389,7 +2451,9 @@ def client(original_function):
if _is_converted_stream_result(_caching_handler_response.cached_result):
logging_obj.stream = True
logging_obj.model_call_details["stream"] = True
return _caching_handler_response.cached_result
return _replayed_when_run_unstreamed(
_caching_handler_response.cached_result, transcription_runs_unstreamed
)
elif _caching_handler_response.embedding_all_elements_cache_hit is True:
return _caching_handler_response.final_embedding_cached_response
@ -2452,7 +2516,7 @@ def client(original_function):
raise
end_time = datetime.datetime.now()
streaming_requested: Final = _is_streaming_request(kwargs=kwargs, call_type=call_type)
streaming_requested: Final = _returns_requested_stream(kwargs, call_type, result)
if streaming_requested or _is_converted_stream_result(result):
logging_obj.stream = True
logging_obj.model_call_details["stream"] = True
@ -2466,6 +2530,15 @@ def client(original_function):
chunks.append(chunk)
return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None))
else:
if call_type == CallTypes.atranscription.value and isinstance(result, openai.AsyncStream):
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
litellm_exception_mapper,
wrap_transcription_stream,
)
result = wrap_transcription_stream(
result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details)
)
_update_response_metadata(
result=result,
logging_obj=logging_obj,
@ -2492,7 +2565,9 @@ def client(original_function):
_call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type)
if _call_type_enum is not None:
result = await async_post_call_success_deployment_hook(
request_data=call_kwargs,
request_data=_guardrail_view_of_requested_stream(call_kwargs)
if transcription_runs_unstreamed
else call_kwargs,
response=result,
call_type=_call_type_enum,
)
@ -2543,7 +2618,7 @@ def client(original_function):
is_litellm_internal_call=_is_litellm_internal_call,
)
return result
return _replayed_when_run_unstreamed(result, transcription_runs_unstreamed)
except Exception as e:
traceback_exception: Final = traceback.format_exc()
# Reuse the timestamp taken right when the deployment call itself failed, before
@ -4151,6 +4226,54 @@ def _register_on_current_map(
_runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload
_runtime_registration_order: Final[dict[str, int]] = {} # mutable-ok: tells which of a row's registered names came last
_runtime_registered_names: Final[dict[str, tuple[str, ...]]] = {} # mutable-ok: names registered onto each live row
_REGISTRATION_TICKS: Final = itertools.count()
@dataclass(frozen=True, slots=True)
class _RegistrationsOverCatalog(Mapping[str, Mapping[str, object]]):
catalog: Mapping[str, Mapping[str, object]]
registrations: Mapping[str, Mapping[str, object]]
registration_order: Mapping[str, int]
registered_names: Mapping[str, tuple[str, ...]]
def __getitem__(self, key: str) -> Mapping[str, object]:
catalog_entry: Final = self.catalog.get(key)
names: Final = dict.fromkeys((*self.registered_names.get(key, ()), key))
present: Final = tuple(name for name in names if name in self.registrations)
if not present:
return self.catalog[key]
oldest_first: Final = sorted(present, key=lambda name: self.registration_order.get(name, -1))
registered: Final = (self.registrations[name] for name in oldest_first)
return MappingProxyType(
dict(itertools.chain.from_iterable(entry.items() for entry in (catalog_entry or {}, *registered)))
)
def __iter__(self) -> Iterator[str]:
return iter({**dict.fromkeys(self.catalog), **dict.fromkeys(self.registrations)})
def __len__(self) -> int:
return len(self.catalog.keys() | self.registrations.keys())
def _record_registered_name(written_key: str, registered_key: str) -> None:
names: Final = _runtime_registered_names.get(written_key, ())
if registered_key not in names:
_runtime_registered_names[written_key] = (*names, registered_key)
def catalog_with_runtime_registrations() -> Mapping[str, Mapping[str, object]]:
"""The loaded catalog plus ``register_model`` calls, without what routers register for their deployments."""
catalog: Final = GetModelCostMap.loaded_model_cost_map()
if not _runtime_registered_model_cost:
return catalog
return _RegistrationsOverCatalog(
catalog,
MappingProxyType(_runtime_registered_model_cost),
MappingProxyType(_runtime_registration_order),
MappingProxyType(_runtime_registered_names),
)
def runtime_model_cost_registrations(
@ -4216,7 +4339,14 @@ def reapply_runtime_model_cost_registrations() -> None:
if _LiveDeploymentReplay.callback is not None:
_LiveDeploymentReplay.callback()
if _runtime_registered_model_cost:
register_model(model_cost=dict(_runtime_registered_model_cost))
register_model(
model_cost=dict(
sorted(
_runtime_registered_model_cost.items(),
key=lambda registration: _runtime_registration_order.get(registration[0], -1),
)
)
)
def cost_map_omits_token_price(*keys: object) -> bool:
@ -4290,9 +4420,12 @@ def register_model(
dict(_runtime_registered_model_cost.get(_registered_key, {})),
_RATE_TABLE.validate_python(_registered_value),
)
_runtime_registration_order[_registered_key] = next(_REGISTRATION_TICKS)
for key, value in loaded_model_cost.items():
write = _register_on_current_map(str(key), value, custom_llm_provider, warning_display_name)
if persist_across_reloads:
_record_registered_name(write.key, str(key))
# Invalidate case-insensitive lookup map since model_cost was modified
_invalidate_model_cost_lowercase_map()
@ -4333,40 +4466,61 @@ def _get_non_default_params(passed_params: dict, default_params: dict, additiona
return non_default_params
_OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS: Final = frozenset(("languages", "keywords", "stream"))
def get_optional_params_transcription(
model: str,
custom_llm_provider: str,
language: str | None = None,
languages: Sequence[str] | None = None,
keywords: Sequence[str] | None = None,
prompt: str | None = None,
response_format: str | None = None,
temperature: int | None = None,
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
stream: bool | None = None,
drop_params: bool | None = None,
**kwargs,
):
from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS
# retrieve all parameters passed to the function
passed_params: Final = locals()
passed_params: Final = locals().copy()
passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS")
passed_params.pop("custom_llm_provider")
passed_params.pop("drop_params")
drop_params = normalize_drop_params(drop_params)
passed_params.pop("model")
custom_llm_provider = passed_params.pop("custom_llm_provider")
drop_params = normalize_drop_params(passed_params.pop("drop_params"))
special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs")
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
transcription_stream_flag,
transcription_stream_unset,
)
for k, v in special_params.items():
passed_params[k] = v
default_params: Final = {
"language": None,
"prompt": None,
"response_format": None,
"temperature": None, # openai defaults this to 0
"timestamp_granularities": None,
}
sends_through_openai_sdk: Final = custom_llm_provider in ("openai", "azure")
drops_unsupported_params: Final = drop_params is True or litellm.drop_params is True
transcription_params: Final = (
OPENAI_TRANSCRIPTION_PARAMS
if sends_through_openai_sdk or drops_unsupported_params
else [param for param in OPENAI_TRANSCRIPTION_PARAMS if param not in _OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS]
)
default_params: Final = {param: None for param in (*transcription_params, "prompt", "temperature")}
stream_unset: Final = transcription_stream_unset(stream)
malformed_stream: Final = not stream_unset and transcription_stream_flag(stream) is None
if malformed_stream and not drops_unsupported_params:
raise UnsupportedParamsError(
status_code=400,
message="`stream` must be a boolean. To drop it from the call, set `litellm.drop_params = True`.",
)
non_default_params: Final = {
k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])
k: v
for k, v in passed_params.items()
if k in default_params and v != default_params[k] and not (k == "stream" and (stream_unset or malformed_stream))
}
optional_params = {}
@ -4375,9 +4529,7 @@ def get_optional_params_transcription(
if len(non_default_params.keys()) > 0:
keys: Final = list(non_default_params.keys())
for k in keys:
if (
drop_params is True or litellm.drop_params is True
) and k not in supported_params: # drop the unsupported non-default values
if drops_unsupported_params and k not in supported_params:
non_default_params.pop(k, None)
elif k not in supported_params:
raise UnsupportedParamsError(
@ -4431,9 +4583,12 @@ def get_optional_params_transcription(
optional_params=optional_params,
passed_params=passed_params,
custom_llm_provider=custom_llm_provider,
openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS,
openai_params=list(default_params) if custom_llm_provider == "bedrock" else transcription_params,
additional_drop_params=kwargs.get("additional_drop_params", None),
)
extra_body: Final = optional_params.get("extra_body")
if isinstance(extra_body, dict) and not extra_body:
optional_params.pop("extra_body")
return optional_params
@ -6006,6 +6161,14 @@ EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"})
LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",))
def header_auth_params(custom_llm_provider: str) -> frozenset[str]:
if custom_llm_provider != "azure_ai":
return frozenset()
from litellm.llms.azure_ai.common_utils import AZURE_ENTRA_CREDENTIAL_PARAM_KEYS
return AZURE_ENTRA_CREDENTIAL_PARAM_KEYS
def add_provider_specific_params_to_optional_params(
optional_params: dict,
passed_params: dict,
@ -6040,8 +6203,10 @@ def add_provider_specific_params_to_optional_params(
**extra_body,
}
dropped_keys: Final = EXTRA_BODY_ROUTING_KEYS | frozenset(
param for param in (additional_drop_params or ()) if isinstance(param, str)
dropped_keys: Final = (
EXTRA_BODY_ROUTING_KEYS
| header_auth_params(custom_llm_provider)
| frozenset(param for param in (additional_drop_params or ()) if isinstance(param, str))
)
ensure_extra_body_is_safe: Final = litellm_utils.ensure_extra_body_is_safe
optional_params["extra_body"] = ensure_extra_body_is_safe(
@ -7182,6 +7347,7 @@ def get_model_info_helper(
"cache_read_input_audio_token_cost_batches", None
),
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
"cache_read_input_token_cost_above_200k_tokens", None
@ -8903,6 +9069,9 @@ def bundled_provider_models(provider: str) -> tuple[str, ...]:
return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {})))
AZURE_MODEL_PLACEHOLDER: Final = "Azure-LLM"
def get_valid_models(
check_provider_endpoint: bool | None = None,
custom_llm_provider: str | None = None,
@ -8958,7 +9127,7 @@ def get_valid_models(
continue
if provider == "azure":
valid_models.append("Azure-LLM")
valid_models.append(AZURE_MODEL_PLACEHOLDER)
elif provider_config is not None and check_provider_endpoint and provider is not None:
valid_models.extend(
_get_valid_models_from_provider_api(
@ -10239,6 +10408,7 @@ class ProviderConfigManager:
def get_provider_audio_transcription_config(
model: str,
provider: LlmProviders,
base_model: str | None = None,
) -> BaseAudioTranscriptionConfig | None:
model_cost_entry: Final = _get_model_cost_entry_for_provider_config(
model=model,
@ -10266,7 +10436,13 @@ class ProviderConfigManager:
return XAIAudioTranscriptionConfig()
elif litellm.LlmProviders.OPENAI == provider:
if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model):
if "gpt-transcribe" in (model, base_model):
from litellm.llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTTranscribeAudioTranscriptionConfig,
)
return OpenAIGPTTranscribeAudioTranscriptionConfig()
elif "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model):
return litellm.OpenAIGPTAudioTranscriptionConfig()
else:
return litellm.OpenAIWhisperAudioTranscriptionConfig()

266
model_prices_and_context_window.json
View file

@ -1,4 +1,168 @@
{
"azure/gpt-realtime-2.1-2026-07-07": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_token_cost": 4e-07,
"input_cost_per_audio_token": 3.2e-05,
"input_cost_per_image_token": 5e-06,
"input_cost_per_token": 4e-06,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"text",
"image",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"cache_read_input_image_token_cost": 5e-07,
"supports_prompt_caching": true,
"supports_reasoning": true
},
"azure/gpt-realtime-2.1-mini-2026-07-07": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image_token": 8e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"text",
"image",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"cache_read_input_image_token_cost": 8e-08,
"supports_prompt_caching": true,
"supports_reasoning": true
},
"azure/gpt-realtime-translate-2026-05-06": {
"input_cost_per_second": 0.0005666666666666667,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"audio",
"text"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_native_streaming": true
},
"azure/gpt-realtime-translate-2026-05-07": {
"input_cost_per_second": 0.0005666666666666667,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://ai.azure.com/catalog/models/gpt-realtime-translate",
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"audio",
"text"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_native_streaming": true
},
"azure/gpt-realtime-whisper-2026-05-06": {
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
"/v1/realtime/transcription_sessions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"max_input_tokens": 32000,
"max_output_tokens": 4096
},
"azure/gpt-realtime-whisper-2026-05-07": {
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
"/v1/realtime/transcription_sessions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true,
"max_input_tokens": 32000,
"max_output_tokens": 4096
},
"sample_spec": {
"code_interpreter_cost_per_session": 0.0,
"computer_use_input_cost_per_1k_tokens": 0.0,
@ -6424,6 +6588,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6458,6 +6625,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6491,6 +6661,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6525,6 +6698,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 1.6e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6547,6 +6723,7 @@
"azure/gpt-realtime-2.1": {
"cache_creation_input_audio_token_cost": 4e-07,
"cache_read_input_audio_token_cost": 4e-07,
"cache_read_input_image_token_cost": 5e-07,
"cache_read_input_token_cost": 4e-07,
"deprecation_date": "2027-06-25",
"input_cost_per_audio_token": 3.2e-05,
@ -6556,6 +6733,9 @@
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
@ -6575,12 +6755,15 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"azure/gpt-realtime-2.1-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_audio_token_cost": 3e-07,
"cache_read_input_image_token_cost": 8e-08,
"cache_read_input_token_cost": 6e-08,
"deprecation_date": "2027-06-25",
"input_cost_per_audio_token": 1e-05,
@ -6590,6 +6773,9 @@
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule",
@ -6609,6 +6795,8 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
@ -6626,6 +6814,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6659,6 +6850,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supported_endpoints": [
"/v1/realtime"
],
@ -6791,6 +6985,9 @@
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -29238,9 +29435,9 @@
},
"gemini-3-pro-image": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_flex": 1e-07,
"cache_read_input_token_cost_priority": 3.6e-07,
"deprecation_date": "2027-05-28",
@ -39069,6 +39266,9 @@
"max_input_tokens": 128000,
"max_output_tokens": 32000,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://developers.openai.com/api/docs/pricing",
@ -39124,6 +39324,8 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
@ -39160,6 +39362,8 @@
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
@ -43106,18 +43310,18 @@
"mode": "embedding"
},
"mistral/mistral-large-4": {
"cache_read_input_token_cost": 1.4e-07,
"input_cost_per_token": 1.36e-06,
"cache_read_input_token_cost": 7e-08,
"input_cost_per_token": 7e-07,
"litellm_provider": "mistral",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 4.18e-06,
"output_cost_per_token": 2.1e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/mistral-large-4",
"source": "https://docs.mistral.ai/models/mistral-large-4-0",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -43127,18 +43331,18 @@
"supports_vision": true
},
"mistral/mistral-large-4-0": {
"cache_read_input_token_cost": 1.4e-07,
"input_cost_per_token": 1.36e-06,
"cache_read_input_token_cost": 7e-08,
"input_cost_per_token": 7e-07,
"litellm_provider": "mistral",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 4.18e-06,
"output_cost_per_token": 2.1e-06,
"reasoning_effort_levels": [
"none",
"high"
],
"source": "https://docs.mistral.ai/models/mistral-large-4",
"source": "https://docs.mistral.ai/models/mistral-large-4-0",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_prompt_caching": true,
@ -52587,9 +52791,9 @@
},
"vertex_ai/gemini-3-pro-image": {
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_above_200k_tokens": 4e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 1e-07,
"cache_read_input_token_cost_flex": 1e-07,
"cache_read_input_token_cost_priority": 3.6e-07,
"deprecation_date": "2027-05-28",
@ -63405,6 +63609,10 @@
"input_cost_per_second": 7.5e-05,
"litellm_provider": "openai",
"mode": "audio_transcription",
"provider_specific_entry": {
"transcription_json_or_text_only": 1,
"transcription_usage_duration": 1
},
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/audio/transcriptions",
@ -63417,7 +63625,8 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"supports_native_streaming": true
},
"gpt-live-transcribe": {
"input_cost_per_second": 0.000283333333333,
@ -63435,7 +63644,8 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"supports_native_streaming": true
},
"gpt-live-1": {
"input_cost_per_second": 0.000833333333333,
@ -63469,7 +63679,8 @@
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true
"supports_audio_output": true,
"supports_native_streaming": true
},
"claude-mythos-5": {
"supports_anthropic_compaction": true,
@ -71340,6 +71551,9 @@
"mode": "realtime",
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/realtime"
@ -71397,7 +71611,11 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supports_native_streaming": true
},
"azure/gpt-transcribe": {
"deprecation_date": "2028-02-01",
@ -71407,7 +71625,8 @@
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supported_endpoints": [
"/v1/audio/transcriptions",
"/v1/realtime/transcription_sessions"
"/v1/realtime/transcription_sessions",
"/v1/realtime"
],
"supported_modalities": [
"text",
@ -71416,7 +71635,13 @@
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
"supports_audio_input": true,
"provider_specific_entry": {
"realtime_ga_only": 1,
"transcription_json_or_text_only": 1,
"transcription_usage_duration": 1
},
"supports_native_streaming": true
},
"azure/gpt-realtime-translate": {
"deprecation_date": "2027-11-09",
@ -71434,7 +71659,11 @@
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true
"supports_audio_output": true,
"provider_specific_entry": {
"realtime_ga_only": 1
},
"supports_native_streaming": true
},
"aihubmix/agnes-2.5-flash": {
"input_cost_per_token": 3e-08,
@ -72873,7 +73102,8 @@
"max_input_tokens": 65536,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://docs.typesafe.ai/models"
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"typesafe/jev-preview": {
"input_cost_per_token": 4.2e-08,

12
scripts/check_openapi_docs.py
View file

@ -18,6 +18,7 @@ from collections.abc import Iterator, Mapping
from typing import Final, NamedTuple
GATED_METHODS: Final = frozenset({"post"})
SCHEMA_MEDIA_TYPES: Final = ("application/json", "application/sdp")
class Violation(NamedTuple):
@ -30,18 +31,21 @@ def _as_mapping(node: object) -> Mapping[str, object]:
return node if isinstance(node, Mapping) else {}
def _json_schema(container: object) -> object:
def _content_schema(container: object) -> object:
content: Final = _as_mapping(_as_mapping(container).get("content"))
return _as_mapping(content.get("application/json")).get("schema")
return next(
(schema for media_type in SCHEMA_MEDIA_TYPES if (schema := _as_mapping(content.get(media_type)).get("schema"))),
None,
)
def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]:
if not _json_schema(operation.get("requestBody")):
if not _content_schema(operation.get("requestBody")):
yield "OA001"
description: Final = operation.get("description")
if not isinstance(description, str) or not description.strip():
yield "OA002"
if not _json_schema(_as_mapping(operation.get("responses")).get("200")):
if not _content_schema(_as_mapping(operation.get("responses")).get("200")):
yield "OA003"

1
tests/code_coverage_tests/router_code_coverage.py
View file

@ -138,6 +138,7 @@ ignored_function_names = [
"_resolve_pre_call_check_input_tokens", # Tested through the token-count failure pre-call check tests in test_router.py
"_params_neither_dropped_nor_allowed", # Tested through the allowed-or-dropped param pre-call check tests in test_router.py
"_deployment_rejects_params", # Tested through the unsupported-param pre-call check tests in test_router.py
"_cache_flags_left_to_the_catalog", # Tested through the wildcard prompt cache flag tests in test_anthropic_cache_control_hook.py
"_deployment_lacks_params", # Tested through the unsupported-param pre-call check tests in test_router.py
"_pre_call_check_requested_chat_params", # Tested through the unsupported-param pre-call check tests in test_router.py
"_deployment_supports_param", # Tested through the deployment-capability pre-call check tests in test_router.py

6
tests/e2e/coverage_registry/llm_nonconversational.yaml
View file

@ -77,6 +77,10 @@
- {id: llm.bedrock_native.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native invoke stream"}
- {id: llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock invoke missing fields and invalid temperature"}
- {id: llm.ocr.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: ocr, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.13 / LIT-4778", rationale: "OCR missing document rejected"}
- {id: llm.realtime.openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: translation, streaming: stream, assertions: [works], source: "realtime_endpoints/endpoints.py", rationale: "Dedicated translation client-secret, raw SDP, and WebSocket paths emit translated audio and transcript deltas"}
- {id: llm.realtime.azure_openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: translation, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA translation session emits translated audio and transcript deltas"}
- {id: llm.realtime.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/realtime/handler.py", rationale: "gpt-live-transcribe and gpt-realtime-whisper emit live transcript deltas"}
- {id: llm.realtime.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA live transcription emits transcript deltas"}
- {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"}
- {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"}
- {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"}
@ -94,6 +98,8 @@
- {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"}
- {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"}
- {id: llm.audio_transcriptions.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.7 / LIT-4778", rationale: "Transcription empty file and missing model are rejected"}
- {id: llm.audio_transcriptions.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "gpt-transcribe streams typed transcript delta and done events"}
- {id: llm.audio_transcriptions.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure gpt-transcribe streams typed transcript delta and done events over the v1 API"}
- {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"}
- {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"}
- {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"}

11
tests/e2e/coverage_registry/schema.py
View file

@ -93,6 +93,8 @@ LlmCapability = Literal[
"tool_search_history",
"tool_use",
"upstream_stream_failure",
"transcription",
"translation",
"vision",
"web_search",
"web_search_server_tool",
@ -162,14 +164,7 @@ class OtherCell(_Base):
Cell = Annotated[
LlmCell
| MgmtCell
| McpCell
| ReliabilityCell
| QuotaCell
| LoggingCell
| GuardrailCell
| OtherCell,
LlmCell | MgmtCell | McpCell | ReliabilityCell | QuotaCell | LoggingCell | GuardrailCell | OtherCell,
Field(discriminator="module"),
]

9
tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md
View file

@ -19,6 +19,15 @@ a matching `function_call` output item, the test sends a `function_call_output`
back, and the follow-up response incorporates the result (the temperature 72
appears). That raw-websocket tool path is the source of truth for tool calling.
The audio surfaces provision their own OpenAI and Azure deployments.
`TestRealtimeTranslation` streams the spoken weather question into
`/v1/realtime/translations` with French as the target and asserts translated
audio, a French transcript, and a billed spend row. `TestRealtimeLiveTranscription`
streams it into `/v1/realtime?intent=transcription` on gpt-live-transcribe and
asserts the deltas add up to the completed transcript and the session is billed.
`test_azure_ga_text_turn_is_answered_and_billed` guards a plain text turn over
the Azure GA deployment, spend row included
`test_pipecat_tool_smoke` is a realism layer through pipecat for openai, azure,
and gemini only (not vertex_ai: native-audio live is flaky under pipecat tool
calling while raw-ws tools pass; see pipecat-ai/pipecat#2544). Assertions are

99
tests/e2e/llm_translation/realtime/realtime_client.py
View file

@ -29,9 +29,13 @@ from models import LiteLLMParamsBody
_M = TypeVar("_M", bound=BaseModel)
REALTIME_ROUTE: Final = "/v1/realtime"
TRANSLATIONS_ROUTE: Final = "/v1/realtime/translations"
def realtime_ws_url(model: str) -> str:
return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}"
def realtime_ws_url(model: str, *, route: str = REALTIME_ROUTE, intent: str | None = None) -> str:
query: Final = {"model": model} if intent is None else {"model": model, "intent": intent}
return f"{ws_base_url()}{route}?{urlencode(query)}"
@dataclass(frozen=True, slots=True)
@ -183,6 +187,75 @@ class InputAudioBufferAppend(BaseModel):
audio: str
class InputAudioBufferCommit(BaseModel):
type: str = "input_audio_buffer.commit"
class SessionInputAudioBufferAppend(BaseModel):
type: str = "session.input_audio_buffer.append"
audio: str
AudioAppend = InputAudioBufferAppend | SessionInputAudioBufferAppend
class TranslationOutput(BaseModel):
language: str
class TranslationAudio(BaseModel):
output: TranslationOutput
class TranslationSessionConfig(BaseModel):
audio: TranslationAudio
class TranslationSessionUpdate(BaseModel):
type: str = "session.update"
session: TranslationSessionConfig
class PcmFormat(BaseModel):
type: str = "audio/pcm"
rate: int = 24000
class InputTranscription(BaseModel):
model: str
class TranscriptionAudioInput(BaseModel):
format: PcmFormat = PcmFormat()
transcription: InputTranscription
class TranscriptionAudio(BaseModel):
input: TranscriptionAudioInput
class TranscriptionSessionConfig(BaseModel):
type: str = "transcription"
audio: TranscriptionAudio
class TranscriptionSessionUpdate(BaseModel):
type: str = "session.update"
session: TranscriptionSessionConfig
RealtimeClientEvent = (
SessionUpdate
| TranslationSessionUpdate
| TranscriptionSessionUpdate
| ConversationItemCreate
| ResponseCreate
| InputAudioBufferAppend
| InputAudioBufferCommit
| SessionInputAudioBufferAppend
)
def user_message(text: str) -> ConversationItemCreate:
return ConversationItemCreate(
item=MessageItem(content=[InputTextContent(text=text)])
@ -320,7 +393,7 @@ class RealtimeSession:
connection: Connection
@step("Send the realtime event {event.type} over the websocket")
def send(self, event: SessionUpdate | ConversationItemCreate | ResponseCreate | InputAudioBufferAppend) -> None:
def send(self, event: RealtimeClientEvent) -> None:
self.connection.send(event.model_dump_json(by_alias=True, exclude_none=True))
@step("Wait for a {stop_type} event on the realtime websocket")
@ -350,9 +423,9 @@ class RealtimeSession:
@step("Stream the spoken question as mic chunks, then silence until the turn goes idle")
def stream_and_collect(
self,
chunks: Sequence[InputAudioBufferAppend],
chunks: Sequence[AudioAppend],
*,
tail: InputAudioBufferAppend,
tail: AudioAppend,
interval: float,
idle: float,
timeout: float,
@ -363,9 +436,9 @@ class RealtimeSession:
def _live_mic_events(
self,
chunks: Sequence[InputAudioBufferAppend],
chunks: Sequence[AudioAppend],
*,
tail: InputAudioBufferAppend,
tail: AudioAppend,
interval: float,
idle: float,
timeout: float,
@ -408,13 +481,19 @@ class RealtimeClient:
)
return model_name, model_id
@step("Open a /v1/realtime websocket session to {model}")
@step("Open a {route} websocket session to {model}")
@contextmanager
def connect(
self, *, key: str, model: str, timeout: float = 15.0
self,
*,
key: str,
model: str,
route: str = REALTIME_ROUTE,
intent: str | None = None,
timeout: float = 15.0,
) -> Generator[RealtimeSession, None, None]:
with connect(
realtime_ws_url(model),
realtime_ws_url(model, route=route, intent=intent),
additional_headers={"Authorization": f"Bearer {key}"},
open_timeout=timeout,
) as connection:

227
tests/e2e/llm_translation/realtime/test_realtime_e2e.py
View file

@ -4,7 +4,10 @@ Each test opens a websocket through the proxy, speaks the OpenAI GA realtime
event schema, and asserts the proxy normalizes the provider's stream into that
schema: the session lifecycle, the canonical response event sequence with a
reconstructed transcript and usage, and a full tool-call round-trip (call ->
tool result -> a follow-up response that uses the result).
tool result -> a follow-up response that uses the result). The audio tests
stream the spoken weather question into /v1/realtime/translations and into a
/v1/realtime?intent=transcription session, and assert the translated or
transcribed output and the session's spend row.
One GA-speaking client validates every provider; only the model alias changes.
Every provider is provisioned at session start, so a missing realtime alias is a
@ -12,19 +15,28 @@ hard failure, not a skip; once configured, a protocol failure is likewise a hard
failure. See REALTIME_COVERAGE_MATRIX.md.
"""
import base64
import wave
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from models import LiteLLMParamsBody, SpendLogRow
from pydantic import BaseModel
from realtime_client import (
PROVIDERS,
TRANSLATIONS_ROUTE,
ConversationItemCreate,
DeltaEvent,
FunctionCallArgumentsDone,
FunctionCallOutputItem,
FunctionTool,
InputAudioBufferAppend,
InputAudioBufferCommit,
InputTranscription,
JsonSchema,
JsonSchemaProperty,
RealtimeClient,
@ -35,8 +47,18 @@ from realtime_client import (
ResponseDone,
ServerEnvelope,
SessionConfig,
SessionInputAudioBufferAppend,
SessionUpdate,
TranscriptionAudio,
TranscriptionAudioInput,
TranscriptionSessionConfig,
TranscriptionSessionUpdate,
TranslationAudio,
TranslationOutput,
TranslationSessionConfig,
TranslationSessionUpdate,
as_text,
events_of_type,
parse_last,
realtime_model,
transcript,
@ -264,3 +286,204 @@ def test_upstream_handshake_refusal_is_an_error_event_and_policy_close(
assert closed.value.rcvd is not None
assert closed.value.rcvd.code == 1008, closed.value
WEATHER_WAV: Final = Path(__file__).parent / "fixtures" / "weather_question_24k.wav"
PCM_RATE: Final = 24000
CHUNK_SECONDS: Final = 0.1
CHUNK_BYTES: Final = int(PCM_RATE * CHUNK_SECONDS) * 2
SILENCE_B64: Final = base64.b64encode(bytes(CHUNK_BYTES)).decode()
REGISTRY_ROUTES: Final = MappingProxyType({"openai": "openai", "azure": "azure_openai"})
def weather_pcm_chunks() -> tuple[str, ...]:
with wave.open(str(WEATHER_WAV), "rb") as wav:
assert (wav.getnchannels(), wav.getsampwidth(), wav.getframerate()) == (1, 2, PCM_RATE), (
"fixture must be mono PCM16 at 24 kHz"
)
pcm: Final = wav.readframes(wav.getnframes())
return tuple(base64.b64encode(pcm[i : i + CHUNK_BYTES]).decode() for i in range(0, len(pcm), CHUNK_BYTES))
def has_billed_row(rows: list[SpendLogRow]) -> bool:
return any((row.spend or 0) > 0 for row in rows)
def audio_marks(provider: RealtimeProvider, capability: str) -> tuple[pytest.MarkDecorator, ...]:
return (
pytest.mark.covers(f"llm.realtime.{REGISTRY_ROUTES[provider.id]}.{capability}.stream.works"),
meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.REALTIME,
providers=(Provider(provider.id),),
models=(provider.litellm_params.model,),
capabilities=(Capability.AUDIO_INPUT,),
mode=Mode.WEBSOCKET,
)
),
)
TRANSLATION_PROVIDERS: Final = (
RealtimeProvider(
"openai",
"openai-translate",
LiteLLMParamsBody(model="openai/gpt-realtime-translate", api_key="os.environ/OPENAI_API_KEY"),
),
RealtimeProvider(
"azure",
"azure-translate",
LiteLLMParamsBody(
model="azure/gpt-realtime-translate",
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
),
),
)
LIVE_TRANSCRIPTION_PROVIDERS: Final = (
RealtimeProvider(
"openai",
"openai-live-transcribe",
LiteLLMParamsBody(model="openai/gpt-live-transcribe", api_key="os.environ/OPENAI_API_KEY"),
),
RealtimeProvider(
"azure",
"azure-live-transcribe",
LiteLLMParamsBody(
model="azure/gpt-live-transcribe",
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
),
),
)
TRANSLATION_PARAMS: Final = tuple(
pytest.param(p, id=p.id, marks=audio_marks(p, "translation")) for p in TRANSLATION_PROVIDERS
)
LIVE_TRANSCRIPTION_PARAMS: Final = tuple(
pytest.param(p, id=p.id, marks=audio_marks(p, "transcription")) for p in LIVE_TRANSCRIPTION_PROVIDERS
)
AZURE_GA_PROVIDER: Final = next(p for p in PROVIDERS if p.id == "azure")
class TranscriptionCompleted(BaseModel):
type: str
transcript: str
class TestRealtimeTranslation:
@pytest.mark.parametrize("provider", TRANSLATION_PARAMS)
def test_spoken_english_comes_back_as_french_audio_and_transcript_and_is_billed(
self,
client: RealtimeClient,
resources: ResourceManager,
scoped_key: str,
provider: RealtimeProvider,
) -> None:
model, model_id = client.provision(provider)
resources.defer(lambda: client.proxy.delete_model(model_id))
with client.connect(key=scoped_key, model=model, route=TRANSLATIONS_ROUTE) as session:
session.send(
TranslationSessionUpdate(
session=TranslationSessionConfig(audio=TranslationAudio(output=TranslationOutput(language="fr")))
)
)
session.collect_until("session.updated", timeout=30)
events: Final = session.stream_and_collect(
tuple(SessionInputAudioBufferAppend(audio=chunk) for chunk in weather_pcm_chunks()),
tail=SessionInputAudioBufferAppend(audio=SILENCE_B64),
interval=CHUNK_SECONDS,
idle=8,
timeout=60,
)
types: Final = tuple(e.type for e in events)
audio_bytes: Final = sum(
len(base64.b64decode(DeltaEvent.model_validate_json(e.payload).delta))
for e in events_of_type(events, "session.output_audio.delta")
)
assert audio_bytes > 0, f"translation returned no translated audio; types={types}"
translated: Final = "".join(
DeltaEvent.model_validate_json(e.payload).delta
for e in events_of_type(events, "session.output_transcript.delta")
).lower()
assert translated.strip(), f"translation returned no translated text; types={types}"
assert "weather" not in translated, f"the spoken weather question came back untranslated: {translated!r}"
rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row)
assert has_billed_row(rows), f"the translation session was not billed: {rows}"
class TestRealtimeLiveTranscription:
@pytest.mark.parametrize("provider", LIVE_TRANSCRIPTION_PARAMS)
def test_spoken_question_streams_transcript_deltas_that_match_the_completed_transcript(
self,
client: RealtimeClient,
resources: ResourceManager,
scoped_key: str,
provider: RealtimeProvider,
) -> None:
model, model_id = client.provision(provider)
resources.defer(lambda: client.proxy.delete_model(model_id))
with client.connect(key=scoped_key, model=model, intent="transcription") as session:
session.send(
TranscriptionSessionUpdate(
session=TranscriptionSessionConfig(
audio=TranscriptionAudio(
input=TranscriptionAudioInput(transcription=InputTranscription(model=model))
)
)
)
)
session.collect_until("session.updated", timeout=30)
for chunk in weather_pcm_chunks():
session.send(InputAudioBufferAppend(audio=chunk))
session.send(InputAudioBufferCommit())
events: Final = session.collect_until("conversation.item.input_audio_transcription.completed", timeout=60)
completed: Final = TranscriptionCompleted.model_validate_json(events[-1].payload)
assert "weather" in completed.transcript.lower(), (
f"transcript of a spoken weather question does not mention weather: {completed.transcript!r}"
)
streamed: Final = "".join(
DeltaEvent.model_validate_json(e.payload).delta
for e in events_of_type(events, "conversation.item.input_audio_transcription.delta")
)
assert streamed.strip() == completed.transcript.strip(), (
f"streamed deltas {streamed!r} do not add up to the completed transcript {completed.transcript!r}"
)
rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row)
assert has_billed_row(rows), f"the live transcription session was not billed: {rows}"
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.REALTIME,
providers=(Provider.AZURE,),
models=(AZURE_GA_PROVIDER.litellm_params.model,),
mode=Mode.WEBSOCKET,
)
)
def test_azure_ga_text_turn_is_answered_and_billed(
client: RealtimeClient, resources: ResourceManager, scoped_key: str
) -> None:
model, model_id = client.provision(AZURE_GA_PROVIDER)
resources.defer(lambda: client.proxy.delete_model(model_id))
with client.connect(key=scoped_key, model=model) as session:
session.collect_until("session.created", timeout=20)
session.send(SessionUpdate(session=SessionConfig(instructions="Reply with the single word hello.")))
session.collect_until("session.updated", timeout=20)
session.send(user_message("Say the single word hello."))
session.send(ResponseCreate())
events: Final = session.collect_until("response.done", timeout=60)
assert "hello" in transcript(events).lower(), f"Azure GA realtime did not answer the turn: {transcript(events)!r}"
rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row)
assert has_billed_row(rows), f"the Azure GA realtime turn was not billed: {rows}"

106
tests/e2e/llm_translation/test_audio_transcriptions_e2e.py
View file

@ -8,10 +8,13 @@ the shared multipart transport, since the SDK refuses to send them. A model-less
request comes back as one of two 400s depending on whether any wildcard
deployment happens to be registered on the shared proxy, so the assertion
accepts either phrasing and holds both to naming the model as the problem.
Streamed gpt-transcribe (stream=true) on OpenAI and Azure must send transcript
deltas, a done event, and data: [DONE], and bill the call.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Final
@ -20,10 +23,10 @@ from e2e_config import unique_marker
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from e2e_http import UnknownApiError, unwrap
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from models import LiteLLMParamsBody, SpendLogRow
from proxy_client import ProxyClient
from pydantic import BaseModel, Field
from sdk_clients import SdkClients
from sdk_clients import SdkClients, response_header
pytestmark = pytest.mark.e2e
@ -221,3 +224,102 @@ class TestWhisperTranscriptionFormats:
assert all(word.start <= word.end for word in transcript.words), (
f"word timings out of order: {transcript.words}"
)
STREAM_DONE: Final = "[DONE]"
@dataclass(frozen=True, slots=True)
class _StreamingTranscriber:
id: str
registry_route: str
provider: Provider
litellm_params: LiteLLMParamsBody
STREAMING_TRANSCRIBERS: Final = (
_StreamingTranscriber(
"openai",
"openai",
Provider.OPENAI,
LiteLLMParamsBody(model="openai/gpt-transcribe", api_key="os.environ/OPENAI_API_KEY"),
),
_StreamingTranscriber(
"azure",
"azure_openai",
Provider.AZURE,
LiteLLMParamsBody(
model="azure/gpt-transcribe",
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
api_version="v1",
),
),
)
STREAMING_PARAMS: Final = tuple(
pytest.param(
t,
id=t.id,
marks=(
pytest.mark.covers(f"llm.audio_transcriptions.{t.registry_route}.transcription.stream.works"),
meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.AUDIO,
providers=(t.provider,),
models=(t.litellm_params.model,),
mode=Mode.STREAM,
)
),
),
)
for t in STREAMING_TRANSCRIBERS
)
class _TranscriptStreamEvent(BaseModel):
type: str
delta: str = ""
text: str = ""
def _has_billed_row(rows: list[SpendLogRow]) -> bool:
return any((row.spend or 0) > 0 for row in rows)
class TestStreamedAudioTranscriptions:
@pytest.mark.parametrize("transcriber", STREAMING_PARAMS)
def test_stream_sends_transcript_deltas_then_done_and_bills_the_call(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, transcriber: _StreamingTranscriber
) -> None:
model: Final = f"e2e-transcribe-stream-{transcriber.id}-{unique_marker()}"
model_id: Final = proxy.create_model(model, transcriber.litellm_params)
resources.defer(lambda: proxy.delete_model(model_id))
client: Final = sdk.openai(resources.key())
with client.audio.transcriptions.with_streaming_response.create(
model=model, file=(WEATHER_WAV.name, WEATHER_WAV.read_bytes(), "audio/wav"), stream=True
) as response:
content_type: Final = response_header(response.headers, "content-type")
call_id: Final = response_header(response.headers, "x-litellm-call-id")
payloads: Final = tuple(
line.removeprefix("data: ") for line in response.iter_lines() if line.startswith("data: ")
)
assert "text/event-stream" in (content_type or ""), f"stream=true did not stream: {content_type!r}"
assert payloads and payloads[-1] == STREAM_DONE, f"the stream did not end with data: [DONE]: {payloads[-3:]}"
assert payloads.count(STREAM_DONE) == 1, f"data: [DONE] was sent more than once: {payloads}"
events: Final = tuple(_TranscriptStreamEvent.model_validate_json(payload) for payload in payloads[:-1])
deltas: Final = tuple(e.delta for e in events if e.type == "transcript.text.delta")
assert len(deltas) > 1, f"the transcript arrived in {len(deltas)} delta events, not as a stream: {events}"
done: Final = events[-1]
assert done.type == "transcript.text.done", f"the last event before [DONE] is not the done event: {done}"
assert "weather" in done.text.lower(), f"transcript of a spoken weather question lost the word: {done.text!r}"
assert "".join(deltas).strip() == done.text.strip(), (
f"the deltas {deltas} do not add up to the final transcript {done.text!r}"
)
assert call_id, "the streamed response carries no x-litellm-call-id to find its spend log"
rows: Final = proxy.poll_logs_for_request_id(call_id, predicate=_has_billed_row)
assert _has_billed_row(rows), f"the streamed transcription was not billed: {rows}"

291
tests/e2e/llm_translation/test_realtime_http_e2e.py
View file

@ -1,14 +1,18 @@
"""Vendor §9.19: realtime client_secrets + calls HTTP surface (LIT-4778).
Websocket coverage already lives under realtime/; this file pins the HTTP
client-secret mint and the missing-auth contract.
client-secret mint (realtime, translation and transcription sessions), the
translation token scope, and the missing-auth contract.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_http import NoBody, assert_auth_denied, unwrap
from e2e_http import NoBody, PartialBody, UnknownApiError, assert_auth_denied, unwrap
from e2e_metadata import Domain, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
@ -18,6 +22,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
REALTIME_BACKEND = "openai/gpt-realtime"
TRANSLATIONS_CLIENT_SECRETS: Final = "/v1/realtime/translations/client_secrets"
class RealtimeSession(BaseModel):
@ -38,8 +43,65 @@ class RealtimeClientSecretRequest(BaseModel):
session: RealtimeSession | None = None
class TurnDetection(BaseModel):
type: str
class SessionTranscription(BaseModel):
model: str | None = None
class SessionAudioInput(BaseModel):
transcription: SessionTranscription | None = None
turn_detection: TurnDetection | None = None
class SessionAudioOutput(BaseModel):
language: str | None = None
class SessionAudio(BaseModel):
input: SessionAudioInput | None = None
output: SessionAudioOutput | None = None
class TranslationSession(BaseModel):
model: str
audio: SessionAudio
class TranscriptionSession(BaseModel):
type: str = "transcription"
audio: SessionAudio
class SessionClientSecretRequest(BaseModel):
session: TranslationSession | TranscriptionSession
class NullableAudioInput(PartialBody):
turn_detection: TurnDetection | None = None
class NullableAudio(PartialBody):
input: NullableAudioInput
class NullableRealtimeSession(PartialBody):
type: str
model: str
instructions: str | None = None
audio: NullableAudio
class NullableClientSecretRequest(PartialBody):
model: str
session: NullableRealtimeSession
class RealtimeClientSecretSession(BaseModel):
type: str | None = None
audio: SessionAudio | None = None
class RealtimeClientSecretResponse(BaseModel):
@ -49,15 +111,85 @@ class RealtimeClientSecretResponse(BaseModel):
def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
model = f"e2e-realtime-http-{unique_marker()}"
model_id = proxy.create_model(
model,
LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY"),
return _register_backend(
proxy, resources, LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY")
)
def _register_backend(
proxy: ProxyClient, resources: ResourceManager, litellm_params: LiteLLMParamsBody
) -> tuple[str, str]:
model: Final = f"e2e-realtime-http-{unique_marker()}"
model_id: Final = proxy.create_model(model, litellm_params)
resources.defer(lambda: proxy.delete_model(model_id))
return model, resources.key()
@dataclass(frozen=True, slots=True)
class _AudioBackend:
id: str
registry_route: str
provider: Provider
litellm_params: LiteLLMParamsBody
@property
def upstream_model(self) -> str:
return self.litellm_params.model.split("/", 1)[1]
def _openai_backend(model: str) -> _AudioBackend:
return _AudioBackend(
"openai", "openai", Provider.OPENAI, LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY")
)
def _azure_backend(model: str) -> _AudioBackend:
return _AudioBackend(
"azure",
"azure_openai",
Provider.AZURE,
LiteLLMParamsBody(model=model, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY"),
)
TRANSLATION_BACKENDS: Final = (
_openai_backend("openai/gpt-realtime-translate"),
_azure_backend("azure/gpt-realtime-translate"),
)
TRANSCRIPTION_BACKENDS: Final = (
_openai_backend("openai/gpt-live-transcribe"),
_azure_backend("azure/gpt-live-transcribe"),
)
def _subject(backend: _AudioBackend) -> pytest.MarkDecorator:
return meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.REALTIME,
providers=(backend.provider,),
models=(backend.litellm_params.model,),
)
)
TRANSLATION_PARAMS: Final = tuple(
pytest.param(
b,
id=b.id,
marks=(pytest.mark.covers(f"llm.realtime.{b.registry_route}.translation.stream.works"), _subject(b)),
)
for b in TRANSLATION_BACKENDS
)
TRANSCRIPTION_PARAMS: Final = tuple(pytest.param(b, id=b.id, marks=_subject(b)) for b in TRANSCRIPTION_BACKENDS)
def _french_translation_request(model: str) -> SessionClientSecretRequest:
return SessionClientSecretRequest(
session=TranslationSession(model=model, audio=SessionAudio(output=SessionAudioOutput(language="fr")))
)
class TestRealtimeHttp:
@pytest.mark.covers("llm.realtime.openai.basic.nonstream.works")
@meta(
@ -110,3 +242,150 @@ class TestRealtimeHttp:
json=NoBody(),
)
assert_auth_denied(result, "realtime calls missing auth")
class TestRealtimeTranslationClientSecrets:
@pytest.mark.parametrize("backend", TRANSLATION_PARAMS)
def test_translations_route_mints_a_french_translation_session(
self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend
) -> None:
model, key = _register_backend(proxy, resources, backend.litellm_params)
secret: Final = unwrap(
proxy.transport.post(
TRANSLATIONS_CLIENT_SECRETS,
headers=proxy.transport.bearer(key),
json=_french_translation_request(model),
response_type=RealtimeClientSecretResponse,
)
)
assert secret.value, f"client secret value missing: {secret}"
assert secret.session is not None and secret.session.type == "translation", (
f"the translations route did not mint a translation session: {secret.session}"
)
output: Final = secret.session.audio.output if secret.session.audio else None
assert output is not None and output.language == "fr", (
f"the requested target language did not reach the session: {secret.session}"
)
@meta(
Subject(
domain=Domain.PROXY_AUTH,
route=Route.REALTIME,
providers=(Provider.OPENAI,),
models=(TRANSLATION_BACKENDS[0].litellm_params.model,),
)
)
def test_translation_token_is_refused_by_the_realtime_calls_route(
self, proxy: ProxyClient, resources: ResourceManager
) -> None:
model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params)
secret: Final = unwrap(
proxy.transport.post(
TRANSLATIONS_CLIENT_SECRETS,
headers=proxy.transport.bearer(key),
json=_french_translation_request(model),
response_type=RealtimeClientSecretResponse,
)
)
assert secret.value, f"client secret value missing: {secret}"
result: Final = proxy.transport.send(
"/v1/realtime/calls", headers=proxy.transport.bearer(secret.value), json=NoBody()
)
assert result.status_code == 401, (
f"a translation token opened a plain realtime call: {result.status_code} {result.body[:300]}"
)
assert "not valid for this Realtime endpoint" in result.body, (
f"the refusal does not say the token belongs to another endpoint: {result.body[:300]}"
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.REALTIME,
providers=(Provider.OPENAI,),
models=(TRANSLATION_BACKENDS[0].litellm_params.model,),
)
)
def test_translation_session_on_the_realtime_route_is_rejected(
self, proxy: ProxyClient, resources: ResourceManager
) -> None:
model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params)
result: Final = proxy.transport.post(
"/v1/realtime/client_secrets",
headers=proxy.transport.bearer(key),
json=RealtimeClientSecretRequest(model=model, session=RealtimeSession(type="translation", model=model)),
response_type=RealtimeClientSecretResponse,
)
match result:
case UnknownApiError(status_code=400, body=body):
assert "translations endpoint" in body, f"the 400 does not point at the translations route: {body}"
case other:
pytest.fail(f"a translation session on /v1/realtime/client_secrets expected a 400, got {other!r}")
class TestRealtimeTranscriptionClientSecrets:
@pytest.mark.parametrize("backend", TRANSCRIPTION_PARAMS)
def test_transcription_session_runs_on_the_deployment_model(
self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend
) -> None:
model, key = _register_backend(proxy, resources, backend.litellm_params)
secret: Final = unwrap(
proxy.transport.post(
"/v1/realtime/client_secrets",
headers=proxy.transport.bearer(key),
json=SessionClientSecretRequest(
session=TranscriptionSession(
audio=SessionAudio(input=SessionAudioInput(transcription=SessionTranscription(model=model)))
)
),
response_type=RealtimeClientSecretResponse,
)
)
assert secret.value, f"client secret value missing: {secret}"
assert secret.session is not None and secret.session.type == "transcription", (
f"the client secret did not mint a transcription session: {secret.session}"
)
audio_input: Final = secret.session.audio.input if secret.session.audio else None
transcription: Final = audio_input.transcription if audio_input else None
assert transcription is not None and transcription.model == backend.upstream_model, (
f"the session does not transcribe with the deployment's model {backend.upstream_model}: {secret.session}"
)
class TestRealtimeClientSecretNulls:
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.REALTIME,
providers=(Provider.OPENAI,),
models=(REALTIME_BACKEND,),
)
)
def test_top_level_null_is_dropped_and_nested_null_is_kept(
self, proxy: ProxyClient, resources: ResourceManager
) -> None:
model, key = _register(proxy, resources)
secret: Final = unwrap(
proxy.transport.post(
"/v1/realtime/client_secrets",
headers=proxy.transport.bearer(key),
json=NullableClientSecretRequest(
model=model,
session=NullableRealtimeSession(
type="realtime",
model=model,
instructions=None,
audio=NullableAudio(input=NullableAudioInput(turn_detection=None)),
),
),
response_type=RealtimeClientSecretResponse,
)
)
assert secret.value, f"a session with instructions: null was not minted: {secret}"
audio_input: Final = secret.session.audio.input if secret.session and secret.session.audio else None
assert audio_input is not None and "turn_detection" in audio_input.model_fields_set, (
f"the session does not report turn detection: {secret.session}"
)
assert audio_input.turn_detection is None, (
f"turn_detection: null was dropped, so the session fell back to {audio_input.turn_detection}"
)

4
tests/integration/compatibility/test_audio_transcriptions_wire.py
View file

@ -345,10 +345,6 @@ def test_audio_transcription_forwards_nested_and_repeated_bracket_fields_verbati
def test_audio_transcription_stream_returns_server_sent_events(gateway: Gateway) -> None:
pytest.skip(
"BUG: stream=true transcriptions reach the upstream but its SSE body comes back as application/json "
'{"text": "data: ..."}, so OpenAI SDK stream=True callers get no transcript events'
)
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
def respond(request: Request) -> Reply:

11
tests/integration/providers/_mantle_gpt_prompt_cache_support.py
View file

@ -475,10 +475,6 @@ def row_key(request_id: str) -> str:
return upstream_id_of(request_id.split("_cache_hit", 1)[0])
def caller_sees_upstream_id(endpoint: Endpoint, *, stream: bool) -> bool:
return (endpoint, stream) != ("messages", True)
def usage_of(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
usage: Final = payload.get("usage")
return JSON.validate_python(usage) if isinstance(usage, dict) else {}
@ -733,10 +729,9 @@ def marked_cell(gateway: Gateway, endpoint: Endpoint, *, stream: bool) -> None:
assert_answered(outcome, marker)
assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream)
assert_usage(endpoint, outcome.usage)
row: Final = success_row(name, marker)
row: Final = success_row(name, marker, outcome.response_id)
assert_priced_row(row, GPT)
if caller_sees_upstream_id(endpoint, stream=stream):
assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id)
assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id)
if not stream:
assert math.isclose(float(outcome.headers["x-litellm-response-cost"]), expected_spend(GPT), rel_tol=1e-9), (
outcome.headers
@ -901,7 +896,7 @@ def assert_marked_sdk_cell(
only_received(wire), expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream
)
assert_usage(endpoint, outcome.usage)
assert_priced_row(success_row(name, marker), GPT)
assert_priced_row(success_row(name, marker, outcome.response_id), GPT)
HOSTILE_OPTIONS: Final[dict[str, JsonValue]] = {

10
tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py
View file

@ -1251,17 +1251,19 @@ def test_s01_to_s06_an_odd_typed_flag_is_read_as_false_never_as_true(
@pytest.mark.timeout(600)
def test_s10_s11_two_deployments_of_one_arn_share_the_flag_and_a_delete_leaves_it_in_place(gateway: Gateway) -> None:
def test_s10_s11_two_deployments_of_one_arn_keep_their_own_flags_and_a_delete_changes_neither(
gateway: Gateway,
) -> None:
arn: Final = _arn("shared")
with wire_server(_peer()) as wire, gateway.scenario() as scenario:
flagged: Final = _deployment(
gateway, scenario, wire, f"bedrock/{arn}", {"supports_prompt_cache_breakpoint": False}, cleanup=False
)
plain: Final = _deployment(gateway, scenario, wire, f"bedrock/{arn}")
for name in (flagged, plain):
for name, cache_points in ((flagged, 0), (plain, 2)):
outcome, received = _observe_at(gateway, wire, "chat", _chat_body(name, _prompt(), _SYSTEM_AND_USER))
_assert_answered(outcome)
assert received.cache_points == 0, (name, received.body)
assert received.cache_points == cache_points, (name, received.body)
_success_row(outcome.response_id)
flagged_identity: Final = _model_id(flagged)
gateway.post("/model/delete", {"id": flagged_identity})
@ -1276,7 +1278,7 @@ def test_s10_s11_two_deployments_of_one_arn_share_the_flag_and_a_delete_leaves_i
wire.drain()
after, after_received = _observe_at(gateway, wire, "chat", _chat_body(plain, _prompt(), _SYSTEM_AND_USER))
_assert_answered(after)
assert after_received.cache_points == 0, after_received.body
assert after_received.cache_points == 2, after_received.body
_success_row(after.response_id)

2
tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py
View file

@ -1211,6 +1211,6 @@ def test_non_claude_mantle_id_with_a_bare_bedrock_twin_keeps_its_route_and_spend
cost: Final = _mantle_cost(gateway, _LUNA)
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(cost), response.text
_assert_success_row(
_spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "responses", cost
_spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "acompletion", cost
)
assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)]

17
tests/integration/providers/test_openai_chat_wire.py
View file

@ -1,4 +1,3 @@
import base64
import json
import uuid
from collections.abc import Callable
@ -547,14 +546,8 @@ def test_azure_gpt_6_bridged_no_cache_function_requests_each_reach_provider_and_
lambda found: len(found) == 2,
seconds=70,
)
by_response_id: Final = {
(decoded := base64.b64decode(string_value(row["request_id"]).removeprefix("resp_")).decode())
.rsplit("response_id:", 1)[1]: (decoded, row)
for row in rows
}
for response_id in response_ids:
decoded, row = by_response_id[response_id]
assert decoded.startswith("litellm:custom_llm_provider:azure;model_id:"), rows
assert sorted(string_value(row["request_id"]) for row in rows) == sorted(response_ids), rows
for row in rows:
assert (
string_value(row["status"]),
string_value(row["cache_hit"]),
@ -601,11 +594,7 @@ def test_azure_gpt_6_bridged_function_requests_without_cache_field_still_hit_cac
seconds=70,
)
priced, cached = rows
priced_request: Final = base64.b64decode(
string_value(priced["request_id"]).removeprefix("resp_")
).decode()
assert priced_request.startswith("litellm:custom_llm_provider:azure;model_id:"), rows
assert priced_request.endswith(";response_id:resp_cached"), rows
assert string_value(priced["request_id"]) == "resp_cached", rows
assert (
priced["status"],
priced["cache_hit"],

95
tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py Normal file
View file

@ -0,0 +1,95 @@
import json
from collections.abc import Awaitable
from datetime import datetime, timezone
from pathlib import Path
from types import MappingProxyType
from typing import Final
import httpx
import pytest
from integration._support.tls import server_context, write_self_signed_cert
from integration._support.wire import Reply, Request, wire_server
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.realtime_api import main as realtime_main
UPSTREAM_PATHS: Final = MappingProxyType(
{
"client_secrets": "/v1/realtime/client_secrets",
"translation_client_secrets": "/v1/realtime/translations/client_secrets",
"calls": "/v1/realtime/calls",
"translation_calls": "/v1/realtime/translations/calls",
}
)
def _logging(route: str) -> Logging:
return Logging(
model="gpt-realtime",
messages=[],
stream=False,
call_type="acreate_realtime_client_secret",
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id=f"test-realtime-{route}",
function_id=f"test-realtime-{route}",
)
def _route_call(route: str, api_base: str) -> Awaitable[httpx.Response]:
match route:
case "client_secrets":
return realtime_main.acreate_realtime_client_secret.__wrapped__(
model="gpt-realtime",
custom_llm_provider="openai",
api_key="sk-test",
api_base=api_base,
litellm_logging_obj=_logging(route),
)
case "translation_client_secrets":
return realtime_main.acreate_realtime_translation_client_secret.__wrapped__(
model="gpt-realtime-translate",
custom_llm_provider="openai",
api_key="sk-test",
api_base=api_base,
litellm_logging_obj=_logging(route),
)
case "calls":
return realtime_main.arealtime_calls.__wrapped__(
openai_ephemeral_key="ek_test",
sdp_body=b"v=0",
model="gpt-realtime",
custom_llm_provider="openai",
api_base=api_base,
litellm_logging_obj=_logging(route),
)
case _:
return realtime_main.arealtime_translation_calls.__wrapped__(
openai_ephemeral_key="ek_test",
sdp_body=b"v=0",
model="gpt-realtime-translate",
custom_llm_provider="openai",
api_base=api_base,
litellm_logging_obj=_logging(route),
)
def _ephemeral_key(_: Request) -> Reply:
return Reply(body=json.dumps({"value": "ek_test"}).encode())
@pytest.mark.asyncio
@pytest.mark.parametrize("route", tuple(UPSTREAM_PATHS))
async def test_openai_realtime_http_routes_trust_the_ca_bundle_set_in_litellm_ssl_verify(
route: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
certificate, key = write_self_signed_cert(tmp_path)
monkeypatch.setattr(litellm, "ssl_verify", str(certificate))
with wire_server(_ephemeral_key, tls=server_context(certificate, key)) as upstream:
response: Final = await _route_call(route, upstream.url)
targets: Final = tuple(request.target for request in upstream.drain())
assert response.status_code == 200
assert response.json() == {"value": "ek_test"}
assert targets == (UPSTREAM_PATHS[route],)

4
tests/integration/spend/test_responses_bridge_spend_once.py
View file

@ -1,5 +1,4 @@
import json
from base64 import b64decode
from collections.abc import Callable
from hashlib import sha256
from typing import Final
@ -100,8 +99,7 @@ def test_bridged_chat_completion_above_log_offload_threshold_logs_and_charges_on
call_rows: Final = [row for row in rows if row["litellm_call_id"] == call_id]
assert len(call_rows) == 1, rows
assert float(str(call_rows[0]["spend"])) == pytest.approx(EXPECTED_SPEND), rows
logged_id: Final = b64decode(str(call_rows[0]["request_id"]).removeprefix("resp_")).decode()
assert logged_id.startswith("litellm:") and f";response_id:{upstream_id}" in logged_id, rows
assert call_rows[0]["request_id"] == upstream_id, rows
key_spend: Final = eventually(
lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)),

457
tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
View file

@ -1,9 +1,13 @@
import asyncio
import copy
import datetime
import json
import os
import queue
import unittest
from collections.abc import Mapping, Sequence
from collections.abc import AsyncIterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast, get_args
from unittest.mock import ANY, MagicMock, Mock, patch
@ -24,15 +28,15 @@ import litellm
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT, ResponsesAPIResponse
from litellm.types.utils import ModelResponse, StandardLoggingPayload, TextCompletionResponse
if TYPE_CHECKING:
from openai.types.responses import ResponseOutputItem
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
def test_completion_over_responses_maps_anthropic_search_tool_to_openai_tool_search() -> None:
response_body: Final = {
@ -191,6 +195,417 @@ def test_auto_summary_sends_the_level_the_target_accepts(
assert request_body["reasoning"] == {"effort": "low", "summary": expected_summary}
@dataclass(frozen=True, slots=True)
class _LoggedSuccess:
call_type: str
payload: StandardLoggingPayload
response_obj: object
complete_streaming_response: object
class _LoggedCallTypes(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.seen_by_caller: Final = MappingProxyType(
{
"completion": queue.SimpleQueue[_LoggedSuccess](),
"acompletion": queue.SimpleQueue[_LoggedSuccess](),
}
)
@staticmethod
def _logged(kwargs: Mapping[str, object], response_obj: object, streaming_key: str) -> _LoggedSuccess:
return _LoggedSuccess(
call_type=str(kwargs["call_type"]),
payload=cast(StandardLoggingPayload, kwargs["standard_logging_object"]),
response_obj=response_obj,
complete_streaming_response=kwargs.get(streaming_key),
)
def log_success_event(
self,
kwargs: Mapping[str, object],
response_obj: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
self.seen_by_caller["completion"].put(self._logged(kwargs, response_obj, "complete_streaming_response"))
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
response_obj: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
self.seen_by_caller["acompletion"].put(
self._logged(kwargs, response_obj, "async_complete_streaming_response")
)
_GET_WEATHER_TOOL: Final = {
"type": "function",
"function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}},
}
_BRIDGED_PROVIDERS: Final = pytest.mark.parametrize(
("model", "url", "provider_kwargs"),
[
("openai/gpt-5.3-codex", "https://api.openai.com/v1/responses", {}),
(
"azure/gpt-6-sol",
"https://example.invalid/openai/responses",
{"api_base": "https://example.invalid", "api_version": "2025-04-01-preview"},
),
("bedrock_mantle/openai.gpt-5.6-terra", _MANTLE_RESPONSES_URL, {"aws_region_name": "us-east-1"}),
],
ids=["openai-responses-only", "azure-tools", "bedrock-mantle-tools"],
)
_PONG_MESSAGE: Final[Mapping[str, object]] = {
"type": "message",
"id": "msg_logged_call_type",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "pong", "annotations": []}],
}
def _searched_responses_body(model: str) -> Mapping[str, object]:
return {
"id": "resp_logged_call_type",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": model.rsplit("/", 1)[-1],
"output": [
{
"type": "web_search_call",
"id": "ws_logged_call_type",
"status": "completed",
"action": {"type": "search", "query": "weather"},
},
_PONG_MESSAGE,
],
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
def _responses_sse(response_body: Mapping[str, object]) -> str:
events: Final = (
{"type": "response.created", "response": {**response_body, "status": "in_progress", "output": [], "usage": None}},
{
"type": "response.output_text.delta",
"item_id": "msg_logged_call_type",
"output_index": 1,
"content_index": 0,
"delta": "pong",
},
{"type": "response.completed", "response": response_body},
)
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
def _bridged_request(model: str, provider_kwargs: Mapping[str, object], stream: bool) -> Mapping[str, object]:
return {
"model": model,
"messages": [{"role": "user", "content": "ping"}],
"tools": [_GET_WEATHER_TOOL],
"api_key": "test-key",
"cache": {"no-cache": True},
"stream": stream,
**provider_kwargs,
}
async def _served_chunk_ids(caller: str, request: Mapping[str, object]) -> frozenset[str]:
if caller == "acompletion":
async_stream: Final = await litellm.acompletion(**request)
assert isinstance(async_stream, CustomStreamWrapper)
return frozenset([chunk.id async for chunk in async_stream])
sync_stream: Final = litellm.completion(**request)
assert isinstance(sync_stream, CustomStreamWrapper)
return frozenset(chunk.id for chunk in sync_stream)
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@_BRIDGED_PROVIDERS
@pytest.mark.asyncio
async def test_bridged_chat_call_is_logged_with_the_callers_call_type(
monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object]
) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = _searched_responses_body(model)
request: Final = _bridged_request(model, provider_kwargs, stream=False)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith=url).mock(return_value=httpx.Response(200, json=response_body))
if caller == "acompletion":
await litellm.acompletion(**request)
else:
litellm.completion(**request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert (logged.call_type, logged.payload["call_type"]) == (caller, caller)
assert isinstance(logged.response_obj, ModelResponse)
assert logged.response_obj.id == response_body["id"]
assert logged.response_obj.choices[0].message.content == "pong"
assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong"
assert logged.payload["response_cost"] == litellm.completion_cost(
completion_response=ResponsesAPIResponse.model_validate(response_body), model=model
)
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@_BRIDGED_PROVIDERS
@pytest.mark.asyncio
async def test_bridged_chat_stream_is_logged_as_the_chat_completion_it_streamed(
monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object]
) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = _searched_responses_body(model)
request: Final = _bridged_request(model, provider_kwargs, stream=True)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith=url).mock(
return_value=httpx.Response(
200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"}
)
)
served_ids: Final = await _served_chunk_ids(caller, request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(logged.complete_streaming_response, ModelResponse)
assert logged.response_obj is logged.complete_streaming_response
assert served_ids == {logged.complete_streaming_response.id}
assert logged.complete_streaming_response.choices[0].message.content == "pong"
assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong"
assert logged.payload["response_cost"] == litellm.completion_cost(
completion_response=ResponsesAPIResponse.model_validate(response_body), model=model
)
def _json_message(item_id: str, document: Mapping[str, object]) -> Mapping[str, object]:
return {
"type": "message",
"id": item_id,
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": json.dumps(document), "annotations": []}],
}
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@pytest.mark.asyncio
async def test_bridged_structured_output_call_is_logged_with_the_json_document_the_client_got(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = {
**_searched_responses_body("openai/gpt-5.3-codex"),
"output": [_json_message("msg_first", {"answer": "pong"}), _json_message("msg_second", {"answer": "again"})],
}
request: Final = {
**_bridged_request("openai/gpt-5.3-codex", {}, stream=False),
"response_format": {"type": "json_object"},
}
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=response_body)
)
served: Final = await litellm.acompletion(**request) if caller == "acompletion" else litellm.completion(**request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(served, ModelResponse)
assert isinstance(logged.response_obj, ModelResponse)
served_message: Final = served.choices[0].message
logged_message: Final = logged.response_obj.choices[0].message
assert json.loads(served_message.content) == {"answer": "pong"}
assert logged_message.content == served_message.content
assert logged_message.provider_specific_fields == served_message.provider_specific_fields
assert logged.payload["response"]["choices"][0]["message"]["content"] == served_message.content
async def _logged_text_completion(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> tuple[TextCompletionResponse, _LoggedSuccess, Mapping[str, object]]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = _searched_responses_body("openai/gpt-5.3-codex")
request: Final = {"model": "openai/gpt-5.3-codex", "prompt": "ping", "api_key": "test-key"}
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=response_body)
)
served: Final = (
await litellm.atext_completion(**request) if caller == "acompletion" else litellm.text_completion(**request)
)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(served, TextCompletionResponse)
return served, logged, response_body
@pytest.mark.parametrize(("caller", "call_type"), [("completion", "text_completion"), ("acompletion", "atext_completion")])
@pytest.mark.asyncio
async def test_bridged_text_completion_is_logged_and_priced_under_the_callers_call_type(
monkeypatch: pytest.MonkeyPatch, caller: str, call_type: str
) -> None:
served, logged, response_body = await _logged_text_completion(monkeypatch, caller)
assert (logged.call_type, logged.payload["call_type"]) == (call_type, call_type)
assert getattr(logged.response_obj, "id") == served.id == response_body["id"]
assert logged.payload["response_cost"] == litellm.completion_cost(
completion_response=ResponsesAPIResponse.model_validate(response_body), model="openai/gpt-5.3-codex"
)
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@pytest.mark.asyncio
async def test_bridged_text_completion_is_logged_as_the_text_completion_it_served(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> None:
served, logged, _ = await _logged_text_completion(monkeypatch, caller)
assert isinstance(logged.response_obj, TextCompletionResponse)
assert logged.response_obj.choices[0].text == served.choices[0].text == "pong"
assert logged.payload["response"]["object"] == served.object == "text_completion"
assert logged.payload["response"]["choices"][0]["text"] == "pong"
@pytest.mark.asyncio
async def test_a_bridged_chat_response_waits_on_the_usage_ack_like_any_chat_response(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from tests.unit.proxy.spend_tracking.test_usage_meter import CommitBarrier
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
database: Final = CommitBarrier()
runtime: Final = UsageCaptureRuntime()
runtime.start(database.write, UsageCaptureSettings(batch_size=2))
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True})
logging_obj: Final = Logging(
model="openai/gpt-5.3-codex",
messages=[],
stream=False,
call_type="acompletion",
start_time=datetime.datetime.now(),
litellm_call_id="bridged-chat",
function_id="bridged-chat",
)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=_searched_responses_body("openai/gpt-5.3-codex"))
)
served: Final = await litellm.acompletion(
model="openai/gpt-5.3-codex",
messages=[{"role": "user", "content": "ping"}],
api_key="test-key",
litellm_logging_obj=logging_obj,
)
response: Final = asyncio.create_task(capture_proxy_response(logging_obj, served))
try:
await database.entered.wait()
assert not response.done()
assert database.committed == ()
database.release.set()
await response
assert "bridged-chat:inbound" in database.committed
finally:
database.release.set()
await runtime.stop(1)
await response
def _served_message_start_id(events: Sequence[bytes]) -> object:
lines: Final = b"".join(events).decode().splitlines()
starts: Final = tuple(
json.loads(line.removeprefix("data:")) for line in lines if line.startswith("data:") and '"message_start"' in line
)
assert len(starts) == 1
return starts[0]["message"]["id"]
@pytest.mark.parametrize("stream", [False, True], ids=["non-stream", "stream"])
@pytest.mark.asyncio
async def test_bridged_messages_call_is_logged_as_anthropic_messages_under_the_id_the_client_got(
monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": [_PONG_MESSAGE]}
upstream: Final = (
httpx.Response(200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"})
if stream
else httpx.Response(200, json=response_body)
)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(return_value=upstream)
served: Final = await litellm.anthropic.messages.acreate(
model="openai/gpt-5.3-codex",
max_tokens=64,
messages=[{"role": "user", "content": "ping"}],
api_key="test-key",
stream=stream,
)
served_id: Final = (
_served_message_start_id([event async for event in cast(AsyncIterator[bytes], served)])
if stream
else cast(Mapping[str, object], served)["id"]
)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller["acompletion"].get, timeout=10)
assert (logged.call_type, logged.payload["call_type"]) == ("anthropic_messages", "anthropic_messages")
assert logged.payload["id"] == served_id
assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong"
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@pytest.mark.asyncio
async def test_bridged_chat_stream_keeps_the_upstream_headers_when_its_output_cannot_be_translated(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": []}
request: Final = _bridged_request("openai/gpt-5.3-codex", {}, stream=True)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(
200,
text=_responses_sse(response_body),
headers={"content-type": "text/event-stream", "x-request-id": "req_upstream"},
)
)
await _served_chunk_ids(caller, request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(logged.complete_streaming_response, ModelResponse)
assert logged.complete_streaming_response.id == response_body["id"]
assert logged.payload["hidden_params"]["additional_headers"]["llm_provider-x-request-id"] == "req_upstream"
def test_convert_chat_completion_messages_to_responses_api_image_input():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
@ -5482,6 +5897,38 @@ def test_transform_request_strips_the_breakpoint_when_the_served_providers_row_o
]
@pytest.mark.parametrize(
("sibling_flag", "model_info", "keeps_the_breakpoint"),
[
(True, {"id": "flagless"}, False),
(True, {"id": "opted-out", "supports_prompt_cache_breakpoint": False}, False),
(False, {"id": "opted-in", "supports_prompt_cache_breakpoint": True}, True),
],
ids=["flagless", "opted-out", "opted-in"],
)
def test_transform_request_reads_the_routed_deployments_own_flag_not_a_siblings_on_the_shared_key(
monkeypatch: pytest.MonkeyPatch,
sibling_flag: bool,
model_info: Mapping[str, object],
keeps_the_breakpoint: bool,
) -> None:
monkeypatch.setitem(
litellm.model_cost,
"bedrock_mantle/qa-mantle-shared",
{**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": sibling_flag},
)
system_content: Final = _bridged_input_for_served_provider(
"qa-mantle-shared", "bedrock_mantle", model_info=model_info
)[0]["content"]
assert system_content == [
{**_MARKED_SYSTEM_PART, "type": "input_text"}
if keeps_the_breakpoint
else {"type": "input_text", "text": "Stable prefix"}
]
def test_transform_request_strips_the_breakpoint_for_a_served_provider_without_a_row_of_its_own(
monkeypatch: pytest.MonkeyPatch,
) -> None:

1
tests/unit/conftest.py
View file

@ -216,6 +216,7 @@ def _reset_aws_auth_caches() -> None:
def _flush_client_caches() -> None:
litellm.in_memory_llm_clients_cache.flush_cache()
image_handling_module.in_memory_cache.flush_cache()
image_handling_module.pdf_probe_verdicts.flush_cache()
_reset_aws_auth_caches()

0
tests/unit/cookbook/__init__.py Normal file
View file

141
tests/unit/cookbook/test_gpt_realtime_translate.py Normal file
View file

@ -0,0 +1,141 @@
import asyncio
import base64
import json
import wave
from pathlib import Path
from types import SimpleNamespace
from typing import Final, cast
import pytest
from websockets.asyncio.client import ClientConnection
from cookbook import gpt_realtime_translate as translate
@pytest.mark.asyncio
async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.1)
audio: Final = bytes(480)
events: Final = iter(
(
{"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()},
{"type": "error", "error": {"message": "session closed"}},
)
)
async def recv() -> str:
event: Final = next(events)
if event["type"] == "session.output_audio.delta":
await asyncio.sleep(0.03)
return json.dumps(event)
sender_finished: Final = asyncio.Event()
sender_finished.set()
output: Final = tmp_path / "translation.wav"
result: Final = await translate.receive_translation(
cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished
)
assert result == 'Realtime API error: {"message": "session closed"}'
with wave.open(str(output), "rb") as rendered:
assert rendered.readframes(240) == audio
@pytest.mark.asyncio
async def test_long_upload_keeps_waiting_for_first_translated_audio_while_still_sending(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05)
audio: Final = bytes(480)
pending: Final[asyncio.Queue[str]] = asyncio.Queue()
sender_finished: Final = asyncio.Event()
async def speak_after_a_long_pause() -> None:
await asyncio.sleep(0.2)
await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}))
await asyncio.sleep(0.02)
sender_finished.set()
await pending.put(json.dumps({"type": "session.closed"}))
output: Final = tmp_path / "translation.wav"
speaker: Final = asyncio.create_task(speak_after_a_long_pause())
result: Final = await translate.receive_translation(
cast(ClientConnection, SimpleNamespace(recv=pending.get)), output, sender_finished
)
await speaker
assert result is None
with wave.open(str(output), "rb") as rendered:
assert rendered.readframes(240) == audio
@pytest.mark.asyncio
async def test_a_quiet_gap_after_the_sender_stops_keeps_the_rest_of_the_translation(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 1.0)
first: Final = bytes(480)
second: Final = bytes(range(240)) * 2
events: Final = iter(
(
{"type": "session.output_audio.delta", "delta": base64.b64encode(first).decode()},
{"type": "session.output_audio.delta", "delta": base64.b64encode(second).decode()},
{"type": "session.closed"},
)
)
async def recv() -> str:
event: Final = next(events)
if event.get("delta") == base64.b64encode(second).decode():
await asyncio.sleep(0.3)
return json.dumps(event)
sender_finished: Final = asyncio.Event()
sender_finished.set()
output: Final = tmp_path / "translation.wav"
result: Final = await translate.receive_translation(
cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished
)
assert result is None
with wave.open(str(output), "rb") as rendered:
assert rendered.readframes(480) == first + second
@pytest.mark.asyncio
async def test_a_stream_that_never_closes_the_session_reports_an_error(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05)
audio: Final = bytes(480)
pending: Final[asyncio.Queue[str]] = asyncio.Queue()
await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}))
sender_finished: Final = asyncio.Event()
sender_finished.set()
result: Final = await translate.receive_translation(
cast(ClientConnection, SimpleNamespace(recv=pending.get)), tmp_path / "translation.wav", sender_finished
)
assert result == "The translation stream went quiet for 0.05 s without closing the session"
@pytest.mark.asyncio
async def test_the_sender_closes_the_session_after_the_last_audio_chunk() -> None:
sent: Final[asyncio.Queue[str]] = asyncio.Queue()
finished: Final = asyncio.Event()
pcm: Final = bytes(translate.CHUNK_BYTES + 2)
await translate.send_audio(cast(ClientConnection, SimpleNamespace(send=sent.put)), pcm, 0.0, finished)
messages: Final = tuple(json.loads(sent.get_nowait()) for _ in range(sent.qsize()))
assert [message["type"] for message in messages] == [
"session.input_audio_buffer.append",
"session.input_audio_buffer.append",
"session.close",
]
assert b"".join(base64.b64decode(message["audio"]) for message in messages[:-1]) == pcm
assert finished.is_set()

763
tests/unit/integrations/test_anthropic_cache_control_hook.py
View file

@ -8,14 +8,18 @@ import subprocess
import sys
import textwrap
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, List, Optional, Tuple, cast
from unittest.mock import MagicMock, patch
import pytest
import respx
from httpx import Response
from openai.types.chat import ChatCompletionMessage
from pydantic import BaseModel, ConfigDict
import litellm
from litellm import Router
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
CARRY_UNMATCHED_MESSAGE_POINTS,
@ -1397,6 +1401,109 @@ def _marks_the_route_sends(model: str, custom_llm_provider: str | None, messages
_SECOND_SEARCH_CALL: Final = {**_SEARCH_CALL, "id": "srvtoolu_02Search"}
_CLIENT_MARKED_CONVERSATION: Final = (
{"role": "system", "content": [{"type": "text", "text": "Handbook", "prompt_cache_breakpoint": {"mode": "explicit"}}]},
{"role": "user", "content": [{"type": "text", "text": "First", "prompt_cache_breakpoint": {"mode": "explicit"}}]},
{"role": "assistant", "content": "First answer"},
{"role": "user", "content": "Second"},
)
def _hook_result(
model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object
) -> tuple[list[AllMessageValues], Mapping[str, object]]:
_, processed, returned_params = AnthropicCacheControlHook().get_chat_completion_prompt(
model=model,
messages=cast(list[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_CONVERSATION))),
non_default_params={"cache_control_injection_points": list(points), **params},
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
return processed, returned_params
def _hook_output(
model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object
) -> list[AllMessageValues]:
return _hook_result(model, points, **params)[0]
def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_claude():
request: Final = AnthropicConfig().transform_request(
model="claude-sonnet-5-5",
messages=_hook_output("anthropic/claude-sonnet-5-5"),
optional_params={"max_tokens": 64},
litellm_params={},
headers={},
)
assert _marks_in(request["system"]) == 0
assert [_marks_in(message) for message in request["messages"]] == [0, 0, 1]
assert "prompt_cache_breakpoint" not in json.dumps(request)
@pytest.mark.parametrize(
"sibling_model_infos",
[
pytest.param((), id="one-deployment"),
pytest.param(
({"supports_prompt_cache_breakpoint": True, "base_model": "anthropic.claude-unmapped-v1:0"},),
id="beside-a-deployment-of-the-same-profile-with-a-base-model",
),
],
)
@pytest.mark.parametrize("route", ["bedrock", "bedrock/converse"])
def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_a_flagged_bedrock_profile(
local_model_cost_map, sibling_model_infos: tuple[Mapping[str, object], ...], route: str
):
from litellm import Router
model: Final = f"{route}/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged"
Router(
model_list=[
{
"model_name": f"claude-profile-{position}",
"litellm_params": {"model": model, "aws_region_name": "us-east-1"},
"model_info": model_info,
}
for position, model_info in enumerate(({"supports_prompt_cache_breakpoint": True}, *sibling_model_infos))
]
)
processed, params = _hook_result(model)
assert "prompt_cache_breakpoint" not in json.dumps(processed)
assert "prompt_cache_options" not in params
assert [_marks_in(message) for message in processed] == [0, 0, 0, 1]
def test_tool_config_injection_point_leaves_client_prompt_cache_breakpoints_on_claude():
processed: Final = _hook_output("anthropic/claude-sonnet-5-5", points=({"location": "tool_config"},))
assert [message["content"] for message in processed[:2]] == [
message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2]
]
@pytest.mark.parametrize(
("model", "params"),
[
pytest.param("openai/gpt-5.6", {"api_base": "https://gateway.example.com/v1"}, id="reads-them"),
pytest.param("openai/gpt-4o", {}, id="openai-model-that-predates-them"),
pytest.param("hosted_vllm/qwen3", {"api_base": "http://vllm.example.com/v1"}, id="self-hosted"),
pytest.param("azure_ai/qa-unflagged-gpt", {"api_base": "https://qa.services.ai.azure.com/models"}, id="foundry"),
],
)
def test_cache_control_injection_points_keep_client_prompt_cache_breakpoints_on_a_route_that_does_not_translate_them(
model: str, params: Mapping[str, object]
):
processed: Final = _hook_output(model, **params)
assert [message["content"] for message in processed[:2]] == [
message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2]
]
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[
@ -4961,6 +5068,33 @@ class TestMalformedInjectionPointsAreIgnored:
assert kwargs == {"cache_control_injection_points": value}
def test_capping_thousands_of_marks_on_one_tool_result_counts_the_request_a_logarithmic_number_of_times():
def marked_parts(prefix: str, count: int) -> list[dict]:
return [
{"type": "text", "text": f"{prefix} {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(count)
]
messages: Final = [
{"role": "user", "content": marked_parts("part", 4)},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": marked_parts("result", 2000)},
]
sent_breakpoints: Final = MagicMock(side_effect=AnthropicCacheControlHook.count_converse_cache_points)
capped: Final = AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
cast(List[AllMessageValues], messages), True, sent_breakpoints, lambda: 0
)
assert [part.get("cache_control") for part in capped[0]["content"]] == [{"type": "ephemeral"}] * 3 + [None]
assert ["cache_control" in part for part in capped[2]["content"]] == [False] * 1997 + [True] * 3
assert sent_breakpoints.call_count < 3 * 20
def test_cache_control_hook_leaves_the_duplicate_tool_result_warning_to_the_request_builder(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
):
@ -5055,6 +5189,635 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi
]
_SHARED_FOUNDRY_MODEL: Final = "azure_ai/qa-shared-gpt-deployment"
_SIBLING_BREAKPOINT_FLAGS: Final = MappingProxyType(
{
"opted-in": MappingProxyType({"supports_prompt_cache_breakpoint": True}),
"opted-out": MappingProxyType({"supports_prompt_cache_breakpoint": False}),
}
)
_FOUNDRY_CHAT_ROUTE: Final = (
"/models/chat/completions",
MappingProxyType(
{
"id": "chatcmpl-azure-ai",
"object": "chat.completion",
"created": 1700000000,
"model": "qa-shared-gpt-deployment",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
}
),
)
_FOUNDRY_RESPONSES_ROUTE: Final = (
"/openai/v1/responses",
MappingProxyType(
{
"id": "resp_azure_ai",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "qa-shared-gpt-deployment",
"output": [
{
"type": "message",
"id": "msg_azure_ai",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
}
],
}
),
)
def _foundry_siblings_sharing_one_model_string(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
registration_order: tuple[str, ...],
upstream: tuple[str, Mapping[str, object]],
) -> tuple[Router, Mapping[str, respx.Route]]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
path, reply = upstream
routes: Final = MappingProxyType(
{
host: respx_mock.post(host=f"{host}.services.ai.azure.com", path=path).mock(
return_value=Response(200, json=dict(reply))
)
for host in registration_order
}
)
router: Final = Router(
model_list=[
{
"model_name": host,
"litellm_params": {
"model": _SHARED_FOUNDRY_MODEL,
"api_base": f"https://{host}.services.ai.azure.com/models",
"api_key": "azure-ai-test-key",
"cache_control_injection_points": [{"location": "message", "role": "system"}],
},
"model_info": {"id": f"{host}-id", **_SIBLING_BREAKPOINT_FLAGS.get(host, {})},
}
for host in registration_order
],
num_retries=0,
)
return router, routes
async def _send_unmarked_system_prompt(router: Router, host: str, entrypoint: str) -> None:
chat_messages: Final = [{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}]
match entrypoint:
case "completion":
router.completion(model=host, messages=chat_messages)
case "acompletion":
await router.acompletion(model=host, messages=chat_messages)
case "aanthropic_messages":
await router.aanthropic_messages(
model=host, system="stable", messages=[{"role": "user", "content": "hi"}], max_tokens=16
)
case "aresponses":
await router.aresponses(model=host, input=chat_messages)
@pytest.mark.parametrize(
("entrypoint", "upstream"),
[
("completion", _FOUNDRY_CHAT_ROUTE),
("acompletion", _FOUNDRY_CHAT_ROUTE),
("aanthropic_messages", _FOUNDRY_CHAT_ROUTE),
("aresponses", _FOUNDRY_RESPONSES_ROUTE),
],
ids=["completion", "acompletion", "aanthropic_messages", "aresponses"],
)
@pytest.mark.parametrize(
"registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")]
)
async def test_foundry_deployment_reads_only_its_own_prompt_cache_flag_not_a_siblings_sharing_its_model_string(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
registration_order: tuple[str, ...],
entrypoint: str,
upstream: tuple[str, Mapping[str, object]],
) -> None:
router, routes = _foundry_siblings_sharing_one_model_string(respx_mock, monkeypatch, registration_order, upstream)
for host in registration_order:
await _send_unmarked_system_prompt(router, host, entrypoint)
def sent_prompt_caching(host: str) -> tuple[object, bool]:
body: Final = routes[host].calls.last.request.content
return json.loads(body).get("prompt_cache_options"), b"prompt_cache_breakpoint" in body
assert {host: sent_prompt_caching(host) for host in registration_order} == {
"opted-in": ({"mode": "implicit"}, True),
"opted-out": (None, False),
"unflagged": (None, False),
}
@pytest.mark.parametrize(
("model", "sibling_flag", "expected"),
[("qa-openai-shared", True, False), ("gpt-99.0-qa-shared", False, True)],
ids=["sibling-opt-in-does-not-turn-it-on", "sibling-opt-out-does-not-turn-it-off"],
)
def test_openai_deployment_reads_the_catalog_not_a_siblings_flag_on_the_shared_key(
monkeypatch: pytest.MonkeyPatch, model: str, sibling_flag: bool, expected: bool
) -> None:
sibling_entry: Final = {
"litellm_provider": "openai",
"mode": "chat",
"supports_prompt_cache_breakpoint": sibling_flag,
}
monkeypatch.setitem(litellm.model_cost, model, sibling_entry)
assert (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, "openai", "https://api.openai.com/v1", model_info={"id": "flagless-deployment"}
)
is expected
)
@pytest.mark.parametrize(
("model", "declared", "api_base", "expected"),
[
("qa-openai-shared", True, "https://api.openai.com/v1", True),
("qa-openai-shared", True, "https://compatible-host.example.com/v1", False),
("gpt-99.0-qa-shared", False, "https://api.openai.com/v1", False),
],
ids=["opt-in-on-openai", "opt-in-on-a-compatible-host-stays-off", "opt-out-beats-the-version-rule"],
)
def test_openai_deployment_declared_flag_answers_support_while_the_host_still_decides(
model: str, declared: bool, api_base: str, expected: bool
) -> None:
assert (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, "openai", api_base, model_info={"supports_prompt_cache_breakpoint": declared}
)
is expected
)
@pytest.mark.parametrize(
"registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")]
)
async def test_foundry_responses_drop_params_keeps_a_client_breakpoint_by_the_deployments_own_flag(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, ...]
) -> None:
router, routes = _foundry_siblings_sharing_one_model_string(
respx_mock, monkeypatch, registration_order, _FOUNDRY_RESPONSES_ROUTE
)
client_marked_system_prompt: Final = {
"role": "system",
"content": [{"type": "input_text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}],
}
for host in registration_order:
await router.aresponses(
model=host, input=[client_marked_system_prompt, {"role": "user", "content": "hi"}], drop_params=True
)
assert {
host: b"prompt_cache_breakpoint" in routes[host].calls.last.request.content for host in registration_order
} == {
"opted-in": True,
"opted-out": False,
"unflagged": True,
}
@pytest.mark.parametrize(
("entrypoint", "part_type", "reply"),
[("acompletion", "text", _FOUNDRY_CHAT_ROUTE[1]), ("aresponses", "input_text", _FOUNDRY_RESPONSES_ROUTE[1])],
ids=["chat", "responses"],
)
@pytest.mark.parametrize("registration_order", [("opted-in", "unflagged"), ("unflagged", "opted-in")])
async def test_azure_drop_params_keeps_a_client_breakpoint_by_the_deployments_own_flag_not_a_siblings(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
registration_order: tuple[str, ...],
entrypoint: str,
part_type: str,
reply: Mapping[str, object],
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
routes: Final = MappingProxyType(
{
host: respx_mock.post(host=f"{host}.openai.azure.com").mock(return_value=Response(200, json=dict(reply)))
for host in registration_order
}
)
router: Final = Router(
model_list=[
{
"model_name": host,
"litellm_params": {
"model": "azure/gpt-4.1-qa-shared",
"api_base": f"https://{host}.openai.azure.com",
"api_key": "azure-test-key",
"api_version": "2025-04-01-preview",
},
"model_info": {"id": f"{host}-id", **_SIBLING_BREAKPOINT_FLAGS.get(host, {})},
}
for host in registration_order
],
num_retries=0,
)
marked_system_prompt: Final = {
"role": "system",
"content": [{"type": part_type, "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}],
}
request: Final = [marked_system_prompt, {"role": "user", "content": "hi"}]
for host in registration_order:
match entrypoint:
case "acompletion":
await router.acompletion(model=host, messages=request, drop_params=True)
case "aresponses":
await router.aresponses(model=host, input=request, drop_params=True)
assert {
host: b"prompt_cache_breakpoint" in routes[host].calls.last.request.content for host in registration_order
} == {"opted-in": True, "unflagged": False}
@pytest.mark.parametrize("registration_order", [("opted-in", "unflagged"), ("unflagged", "opted-in")])
async def test_openai_messages_bridge_sends_a_breakpoint_by_the_deployments_own_flag_not_a_siblings(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, ...]
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
_, reply = _FOUNDRY_RESPONSES_ROUTE
responses_route: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=Response(200, json=dict(reply))
)
router: Final = Router(
model_list=[
{
"model_name": name,
"litellm_params": {"model": "openai/qa-shared-gpt-deployment", "api_key": "openai-test-key"},
"model_info": {"id": f"{name}-id", **_SIBLING_BREAKPOINT_FLAGS.get(name, {})},
}
for name in registration_order
],
num_retries=0,
)
async def sends_a_breakpoint(name: str) -> bool:
await router.aanthropic_messages(
model=name,
system=[{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}],
messages=[{"role": "user", "content": "hi"}],
max_tokens=16,
)
return b"prompt_cache_breakpoint" in responses_route.calls.last.request.content
assert {name: await sends_a_breakpoint(name) for name in registration_order} == {
"opted-in": True,
"unflagged": False,
}
_OPENAI_ROUTES: Final = MappingProxyType(
{
"acompletion": ("https://api.openai.com/v1/chat/completions", _FOUNDRY_CHAT_ROUTE[1]),
"aresponses": ("https://api.openai.com/v1/responses", _FOUNDRY_RESPONSES_ROUTE[1]),
"aanthropic_messages": ("https://api.openai.com/v1/responses", _FOUNDRY_RESPONSES_ROUTE[1]),
}
)
def _openai_route(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str) -> respx.Route:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
url, reply = _OPENAI_ROUTES[entrypoint]
return respx_mock.post(url).mock(return_value=Response(200, json=dict(reply)))
@pytest.mark.parametrize("declared", [True, False])
async def test_a_wildcard_deployments_prompt_cache_flag_leaves_each_matched_model_to_the_catalog(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, declared: bool
) -> None:
chat: Final = _openai_route(respx_mock, monkeypatch, "acompletion")
router: Final = Router(
model_list=[
{
"model_name": "openai/*",
"litellm_params": {
"model": "openai/*",
"api_key": "openai-test-key",
"cache_control_injection_points": [{"location": "message", "role": "system"}],
},
"model_info": {"id": "wildcard-id", "supports_prompt_cache_breakpoint": declared},
}
],
num_retries=0,
)
async def sends_prompt_cache_options(model: str) -> bool:
await _send_unmarked_system_prompt(router, model, "acompletion")
return b"prompt_cache_options" in chat.calls.last.request.content
assert {model: await sends_prompt_cache_options(model) for model in ("openai/gpt-4.1-qa", "openai/gpt-99.0-qa")} == {
"openai/gpt-4.1-qa": False,
"openai/gpt-99.0-qa": True,
}
async def test_an_azure_ai_wildcard_deployments_breakpoint_flag_still_speaks_for_each_matched_model(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
path, reply = _FOUNDRY_CHAT_ROUTE
chat: Final = respx_mock.post(host="qa-wildcard.services.ai.azure.com", path=path).mock(
return_value=Response(200, json=dict(reply))
)
router: Final = Router(
model_list=[
{
"model_name": "azure_ai/*",
"litellm_params": {
"model": "azure_ai/*",
"api_base": "https://qa-wildcard.services.ai.azure.com/models",
"api_key": "azure-ai-test-key",
"cache_control_injection_points": [{"location": "message", "role": "system"}],
},
"model_info": {"id": "wildcard-id", "supports_prompt_cache_breakpoint": True},
}
],
num_retries=0,
)
await _send_unmarked_system_prompt(router, "azure_ai/qa-unlisted-alias", "acompletion")
body: Final = chat.calls.last.request.content
assert (json.loads(body).get("prompt_cache_options"), b"prompt_cache_breakpoint" in body) == (
{"mode": "implicit"},
True,
)
@pytest.mark.parametrize("entrypoint", ["acompletion", "aresponses", "aanthropic_messages"])
@pytest.mark.parametrize("declared_in", ["litellm_params", "model_info"])
async def test_a_base_model_that_takes_breakpoints_leaves_an_openai_deployments_request_as_it_was(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str, declared_in: str
) -> None:
upstream: Final = _openai_route(respx_mock, monkeypatch, entrypoint)
def deployment(name: str, base_model: Mapping[str, str]) -> dict[str, object]:
return {
"model_name": name,
"litellm_params": {
"model": "openai/gpt-4.1-qa",
"api_key": "openai-test-key",
"cache_control_injection_points": [{"location": "message", "role": "system"}],
**(base_model if declared_in == "litellm_params" else {}),
},
"model_info": {"id": f"{name}-id", **(base_model if declared_in == "model_info" else {})},
}
router: Final = Router(
model_list=[deployment("plain", {}), deployment("priced-as-breakpoint-model", {"base_model": "gpt-99.0-qa"})],
num_retries=0,
)
async def sent_body(name: str) -> object:
await _send_unmarked_system_prompt(router, name, entrypoint)
return json.loads(upstream.calls.last.request.content)
assert await sent_body("priced-as-breakpoint-model") == await sent_body("plain")
def test_a_routed_call_honors_a_register_model_breakpoint_opt_out_made_under_the_provider_prefixed_name() -> None:
from litellm.utils import catalog_with_runtime_registrations
model: Final = next(
key
for key, entry in catalog_with_runtime_registrations().items()
if entry.get("litellm_provider") == "openai" and entry.get("supports_prompt_cache_breakpoint") is True
)
litellm.register_model({f"openai/{model}": {"supports_prompt_cache_breakpoint": False}})
assert supports_openai_prompt_cache_breakpoint(model, "openai", {"id": "plain-id"}) is False
_APPLICATION_INFERENCE_PROFILE: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/qa"
@pytest.mark.parametrize(
("model", "custom_llm_provider", "expected"),
[
("us.anthropic.claude-sonnet-5-5", "bedrock", False),
("converse/us.anthropic.claude-sonnet-5-5", "bedrock", False),
("invoke/us.anthropic.claude-sonnet-5-5", "bedrock", False),
("converse_like/us.anthropic.claude-sonnet-5-5", "bedrock", False),
("openai/gpt-oss-120b", "bedrock", False),
("moonshotai.kimi-k3", "bedrock", False),
(_APPLICATION_INFERENCE_PROFILE, "bedrock", False),
("claude-sonnet-5-5", "anthropic", False),
("claude-sonnet-5-5", "azure_ai", False),
("claude-sonnet-5-5", "vertex_ai", False),
("gemini-3-pro", "gemini", False),
("qa-custom-model", "qa-custom-handler", False),
("qa-shared-gpt-deployment", "azure_ai", True),
("my-custom-deployment", "azure", True),
("qa-mantle-model", "bedrock_mantle", True),
("anthropic/qa-gateway-claude", "openrouter", False),
("qa-downstream-claude", "litellm_proxy", False),
("qa-self-hosted-model", "hosted_vllm", False),
],
)
def test_a_declared_breakpoint_flag_picks_openais_dialect_only_where_the_request_goes_out_openai_shaped(
model: str, custom_llm_provider: str, expected: bool
) -> None:
assert (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, custom_llm_provider, model_info={"supports_prompt_cache_breakpoint": True}
)
is expected
)
def test_a_gateway_deployment_declaring_the_flag_picks_openais_dialect_for_a_model_the_cost_map_lists_there() -> None:
litellm.register_model({"openrouter/qa-listed-gpt": {"litellm_provider": "openrouter", "mode": "chat"}})
assert [
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, "openrouter", model_info={"supports_prompt_cache_breakpoint": True}
)
for model in ("qa-listed-gpt", "qa-unlisted-gpt")
] == [True, False]
def test_a_gateway_deployment_declaring_the_flag_picks_openais_dialect_when_its_model_info_lists_that_provider() -> None:
assert [
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
"qa-self-hosted-gpt",
"hosted_vllm",
model_info={"supports_prompt_cache_breakpoint": True, "litellm_provider": listed_provider},
)
for listed_provider in ("hosted_vllm", "openrouter")
] == [True, False]
def test_a_custom_handler_provider_declaring_the_flag_keeps_the_anthropic_mark() -> None:
litellm.register_model(
{"qa-custom-handler/qa-model": {"litellm_provider": "qa-custom-handler", "mode": "chat"}}
)
assert (
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
"qa-model", "qa-custom-handler", model_info={"supports_prompt_cache_breakpoint": True}
)
is False
)
def test_a_routed_deployment_without_its_own_flag_reads_a_register_model_opt_in_but_not_a_siblings() -> None:
litellm.register_model(
{"azure_ai/qa-registered-gpt": {"litellm_provider": "azure_ai", "supports_prompt_cache_breakpoint": True}}
)
Router(
model_list=[
{
"model_name": "qa-opted-in-sibling",
"litellm_params": {"model": "azure_ai/qa-sibling-gpt", "api_base": "https://qa.services.ai.azure.com"},
"model_info": {"id": "qa-opted-in-sibling-id", "supports_prompt_cache_breakpoint": True},
}
]
)
assert [
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
model, "azure_ai", model_info={"id": "qa-flagless-id"}
)
for model in ("qa-registered-gpt", "qa-sibling-gpt")
] == [True, False]
@pytest.mark.parametrize(
("entrypoint", "injection_points"),
[
("completion", [{"location": "message", "role": "system"}]),
("acompletion", [{"location": "message", "role": "system"}]),
("aresponses", [{"location": "message", "role": "system"}]),
("aresponses-client-marked", None),
],
)
async def test_an_azure_custom_deployment_declaring_the_breakpoint_flag_gets_openais_breakpoint(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
entrypoint: str,
injection_points: list[dict[str, str]] | None,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
azure: Final = respx_mock.post(host="qa-azure.openai.azure.com").mock(
side_effect=lambda request: Response(
200,
json=dict(
_FOUNDRY_RESPONSES_ROUTE[1] if request.url.path.endswith("/responses") else _FOUNDRY_CHAT_ROUTE[1]
),
)
)
deployment_params: Final = {
"model": "azure/my-custom-deployment",
"api_base": "https://qa-azure.openai.azure.com",
"api_key": "azure-test-key",
"api_version": "2025-04-01-preview",
}
router: Final = Router(
model_list=[
{
"model_name": "custom-azure",
"litellm_params": (
deployment_params
if injection_points is None
else {**deployment_params, "cache_control_injection_points": injection_points}
),
"model_info": {"id": "custom-azure-id", "supports_prompt_cache_breakpoint": True},
}
],
num_retries=0,
)
client_marked_system_prompt: Final = {
"role": "system",
"content": [{"type": "input_text", "text": "stable", "cache_control": {"type": "ephemeral"}}],
}
match entrypoint:
case "aresponses-client-marked":
await router.aresponses(
model="custom-azure", input=[client_marked_system_prompt, {"role": "user", "content": "hi"}]
)
case _:
await _send_unmarked_system_prompt(router, "custom-azure", entrypoint)
body: Final = azure.calls.last.request.content
assert (b"prompt_cache_breakpoint" in body, b"cache_control" in body) == (True, False)
_BEDROCK_CONVERSE_REPLY: Final = MappingProxyType(
{
"output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 5, "outputTokens": 1, "totalTokens": 6},
}
)
_BEDROCK_INVOKE_REPLY: Final = MappingProxyType(
{
"id": "msg_bedrock",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 5, "output_tokens": 1},
}
)
@pytest.mark.parametrize(
("entrypoint", "expected_system"),
[
("completion", [{"text": "stable"}, {"cachePoint": {"type": "default"}}]),
("acompletion", [{"text": "stable"}, {"cachePoint": {"type": "default"}}]),
("aanthropic_messages", [{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}]),
],
)
async def test_a_bedrock_claude_deployment_declaring_the_breakpoint_flag_keeps_bedrocks_own_cache_marks(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str, expected_system: list[object]
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
bedrock: Final = respx_mock.post(host="bedrock-runtime.us-east-1.amazonaws.com").mock(
side_effect=lambda request: Response(
200,
json=dict(_BEDROCK_INVOKE_REPLY if request.url.path.endswith("/invoke") else _BEDROCK_CONVERSE_REPLY),
)
)
router: Final = Router(
model_list=[
{
"model_name": "claude-on-bedrock",
"litellm_params": {
"model": "bedrock/us.anthropic.claude-sonnet-5-5",
"aws_access_key_id": "bedrock-test-key",
"aws_secret_access_key": "bedrock-test-secret",
"aws_region_name": "us-east-1",
"cache_control_injection_points": [{"location": "message", "role": "system"}],
},
"model_info": {"id": "claude-on-bedrock-id", "supports_prompt_cache_breakpoint": True},
}
],
num_retries=0,
)
await _send_unmarked_system_prompt(router, "claude-on-bedrock", entrypoint)
body: Final = bedrock.calls.last.request.content
assert (json.loads(body)["system"], b"prompt_cache" in body) == (expected_system, False)
def _search_conversation_with(follow_up: dict[str, object]) -> list[AllMessageValues]:
return [
cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}),

319
tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py Normal file
View file

@ -0,0 +1,319 @@
import datetime
import io
import math
import wave
from collections.abc import AsyncIterator, Iterator
from typing import Final
import anyio
import anyio.lowlevel
import httpx
import pytest
import respx
from openai import AsyncOpenAI, OpenAI
import litellm
from litellm import Router
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.audio_utils.transcription_streaming import (
litellm_exception_mapper,
wrap_transcription_stream,
)
from litellm.types.utils import TranscriptionResponse
_STARTED_AT: Final = datetime.datetime(2026, 10, 28, 12, 0, 0) # noqa: DTZ001 # callback timestamps use the legacy naive contract
class _FirstDeltaThenStall(httpx.AsyncByteStream):
async def __aiter__(self) -> AsyncIterator[bytes]:
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
await anyio.sleep_forever()
class _FirstDeltaThenFails(httpx.AsyncByteStream):
def __init__(self, failure: httpx.TransportError) -> None:
self._failure = failure
async def __aiter__(self) -> AsyncIterator[bytes]:
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
raise self._failure
def _transcription_router(failing_api_base: str, healthy_api_base: str) -> Router:
return Router(
model_list=[
{
"model_name": "transcriber",
"litellm_params": {
"model": "openai/gpt-transcribe",
"api_key": "sk-test",
"api_base": api_base,
"order": order,
},
}
for order, api_base in ((1, failing_api_base), (2, healthy_api_base))
],
allowed_fails=0,
cooldown_time=60,
)
class _RecordingLogging:
def __init__(self) -> None:
self.logged: tuple[TranscriptionResponse, ...] = ()
def success_handler(
self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime
) -> None:
return None
async def async_success_handler(
self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime
) -> None:
await anyio.lowlevel.checkpoint()
self.logged = (*self.logged, result)
def handle_sync_success_callbacks_for_async_calls(
self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime
) -> None:
return None
def failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
*,
for_async_request: bool = False,
) -> None:
return None
async def async_failure_handler(
self,
exception: Exception,
traceback_exception: str,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
return None
@pytest.mark.asyncio
async def test_a_stream_the_client_drops_mid_transcript_is_still_logged_as_a_success() -> None:
upstream: Final = AsyncOpenAI(
api_key="sk-test",
base_url="https://upstream.example/v1",
http_client=httpx.AsyncClient(
transport=httpx.MockTransport(
lambda request: httpx.Response(
200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenStall()
)
)
),
)
stream: Final = await upstream.audio.transcriptions.create(
model="gpt-transcribe", file=("sample.wav", b"audio"), stream=True
)
logging_obj: Final = _RecordingLogging()
wrapped: Final = wrap_transcription_stream(stream, logging_obj, _STARTED_AT, litellm_exception_mapper({}))
with anyio.CancelScope() as client_connection:
async for _ in wrapped:
client_connection.cancel()
assert [result.text for result in logging_obj.logged] == ["Good"]
@pytest.mark.asyncio
async def test_a_deployment_whose_stream_times_out_mid_transcript_goes_into_cooldown(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router: Final = _transcription_router("https://stalls.example/v1", "https://healthy.example/v1")
with respx.mock(assert_all_called=False) as upstream:
upstream.post("https://stalls.example/v1/audio/transcriptions").mock(
side_effect=lambda request: httpx.Response(
200,
headers={"content-type": "text/event-stream"},
stream=_FirstDeltaThenFails(httpx.ReadTimeout("read timed out")),
)
)
upstream.post("https://healthy.example/v1/audio/transcriptions").mock(
return_value=httpx.Response(
200,
headers={"content-type": "text/event-stream"},
content=b'data: {"type":"transcript.text.done","text":"Good morning"}\n\n',
)
)
stalled: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True)
with pytest.raises(litellm.Timeout):
async for _ in stalled:
pass
await anyio.sleep(0.1)
retried: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True)
events: Final = [event async for event in retried]
assert [getattr(event, "text", None) for event in events] == ["Good morning"]
@pytest.mark.asyncio
async def test_a_stream_the_upstream_drops_mid_transcript_raises_a_litellm_error_to_the_caller() -> None:
upstream: Final = AsyncOpenAI(
api_key="sk-test",
base_url="https://upstream.example/v1",
http_client=httpx.AsyncClient(
transport=httpx.MockTransport(
lambda request: httpx.Response(
200,
headers={"content-type": "text/event-stream"},
stream=_FirstDeltaThenFails(httpx.RemoteProtocolError("peer closed connection")),
)
)
),
)
stream: Final = await litellm.atranscription(
model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream
)
with pytest.raises(litellm.APIConnectionError, match="peer closed connection"):
async for _ in stream:
pass
class _FailureHookCounter(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.failures: tuple[str, ...] = ()
def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
self.failures = (*self.failures, "sync_failure")
async def async_log_failure_event(
self, kwargs: object, response_obj: object, start_time: object, end_time: object
) -> None:
self.failures = (*self.failures, "async_failure")
@pytest.mark.asyncio
async def test_an_async_stream_that_fails_mid_transcript_is_logged_as_one_failure(
monkeypatch: pytest.MonkeyPatch,
) -> None:
counter: Final = _FailureHookCounter()
monkeypatch.setattr(litellm, "callbacks", [counter])
upstream: Final = AsyncOpenAI(
api_key="sk-test",
base_url="https://upstream.example/v1",
http_client=httpx.AsyncClient(
transport=httpx.MockTransport(
lambda request: httpx.Response(
200,
headers={"content-type": "text/event-stream"},
stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")),
)
)
),
)
stream: Final = await litellm.atranscription(
model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream
)
with pytest.raises(litellm.APIConnectionError):
async for _ in stream:
pass
assert counter.failures == ("async_failure",)
class _FirstDeltaThenResetSync(httpx.SyncByteStream):
def __iter__(self) -> Iterator[bytes]:
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
raise httpx.ReadError("connection reset")
class _FailureCostRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.costs: tuple[object, ...] = ()
def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
async def async_log_failure_event(
self, kwargs: object, response_obj: object, start_time: object, end_time: object
) -> None:
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
def _silent_wav(seconds: int) -> bytes:
buffer: Final = io.BytesIO()
with wave.open(buffer, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(16_000)
wav.writeframes(b"\x00\x00" * 16_000 * seconds)
return buffer.getvalue()
def _reset_mid_transcript(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
headers={"content-type": "text/event-stream"},
stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")),
)
def _reset_mid_transcript_sync(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenResetSync())
def _per_second_rate(model: str) -> float:
rate: Final = litellm.model_cost[model]["input_cost_per_second"]
assert isinstance(rate, float)
return rate
@pytest.mark.asyncio
async def test_an_async_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
recorder: Final = _FailureCostRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
upstream: Final = AsyncOpenAI(
api_key="sk-test",
base_url="https://upstream.example/v1",
http_client=httpx.AsyncClient(transport=httpx.MockTransport(_reset_mid_transcript)),
)
stream: Final = await litellm.atranscription(
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
)
with pytest.raises(litellm.APIConnectionError):
async for _ in stream:
pass
assert len(recorder.costs) == 1
assert isinstance(recorder.costs[0], float)
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))
def test_a_sync_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(monkeypatch: pytest.MonkeyPatch) -> None:
recorder: Final = _FailureCostRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
upstream: Final = OpenAI(
api_key="sk-test",
base_url="https://upstream.example/v1",
http_client=httpx.Client(transport=httpx.MockTransport(_reset_mid_transcript_sync)),
)
stream: Final = litellm.transcription(
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
)
with pytest.raises(litellm.APIConnectionError):
for _ in stream:
pass
assert len(recorder.costs) == 1
assert isinstance(recorder.costs[0], float)
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))

Some files were not shown because too many files have changed in this diff Show more