From bd603fa44e0b24c0e11d7ab3fef4e0d904895916 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 12:12:59 -0500 Subject: [PATCH 01/45] feat(realtime): support latest OpenAI audio models --- cookbook/gpt_realtime_translate.py | 242 ++++++++++ litellm/__init__.py | 18 +- litellm/constants.py | 34 ++ litellm/cost_calculator.py | 71 ++- .../audio_utils/transcription_streaming.py | 204 +++++++++ .../litellm_core_utils/llm_cost_calc/utils.py | 79 +++- .../convert_dict_to_response.py | 15 +- .../litellm_core_utils/realtime_streaming.py | 237 ++++++++-- litellm/llms/azure/audio_transcriptions.py | 21 +- litellm/llms/azure/realtime/handler.py | 34 +- .../azure/realtime/http_transformation.py | 21 +- .../base_llm/realtime/http_transformation.py | 10 + litellm/llms/custom_httpx/llm_http_handler.py | 283 ++++++++++-- litellm/llms/openai/realtime/handler.py | 127 +++++- .../openai/realtime/http_transformation.py | 12 + .../transcriptions/gpt_transformation.py | 30 +- litellm/llms/openai/transcriptions/handler.py | 40 +- litellm/main.py | 97 +++- litellm/proxy/_lazy_features.py | 15 + litellm/proxy/_lazy_openapi_snapshot.json | 180 ++++++++ litellm/proxy/_types.py | 13 + litellm/proxy/common_request_processing.py | 2 + .../proxy/common_utils/http_parsing_utils.py | 20 +- litellm/proxy/proxy_server.py | 88 +++- litellm/proxy/realtime_endpoints/endpoints.py | 137 +++++- litellm/proxy/route_llm_request.py | 2 + litellm/realtime_api/README.md | 14 +- litellm/realtime_api/main.py | 284 +++++++++++- litellm/responses/streaming_iterator.py | 2 + litellm/responses/utils.py | 8 +- litellm/router.py | 12 + litellm/types/llms/openai.py | 14 + litellm/types/realtime.py | 50 ++- litellm/types/utils.py | 38 +- litellm/utils.py | 38 +- .../llm_nonconversational.yaml | 6 + tests/e2e/coverage_registry/schema.py | 12 +- .../test_realtime_streaming.py | 188 ++++++++ .../llms/azure/test_azure_common_utils.py | 5 + .../realtime/test_openai_realtime_handler.py | 303 ++++++------- .../realtime/test_transcription_sessions.py | 55 ++- .../llms/openai/realtime/test_translation.py | 265 +++++++++++ .../transcriptions/test_gpt_transcribe.py | 297 +++++++++++++ .../test_transcription_duration_hidden.py | 62 ++- tests/unit/proxy/auth/test_route_checks.py | 31 ++ .../proxy/proxy_server/test_routes_audio.py | 57 ++- .../test_realtime_webrtc_endpoints.py | 414 +++++++++++++++--- ...test_proxy_server_endpoints_and_startup.py | 14 +- .../test_streaming_iterator_error_events.py | 2 + tests/unit/test_utils.py | 3 + 50 files changed, 3725 insertions(+), 481 deletions(-) create mode 100644 cookbook/gpt_realtime_translate.py create mode 100644 litellm/litellm_core_utils/audio_utils/transcription_streaming.py create mode 100644 tests/unit/llms/openai/realtime/test_translation.py create mode 100644 tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py new file mode 100644 index 00000000000..73c91eaa1e9 --- /dev/null +++ b/cookbook/gpt_realtime_translate.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import base64 +import json +import os +import sys +import wave +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode, urlsplit, urlunsplit + +import websockets +from websockets.asyncio.client import ClientConnection + +SAMPLE_RATE = 24_000 +CHANNELS = 1 +SAMPLE_WIDTH = 2 +CHUNK_DURATION_SECONDS = 0.1 +CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS) +OUTPUT_IDLE_TIMEOUT_SECONDS = 3.0 +INITIAL_RESPONSE_TIMEOUT_SECONDS = 30.0 +AUDIO_EVENT_TYPES = frozenset( + { + "session.output_audio.delta", + "response.audio.delta", + "response.output_audio.delta", + } +) +TRANSCRIPT_EVENT_TYPES = frozenset( + { + "session.output_transcript.delta", + "response.text.delta", + "response.output_audio_transcript.delta", + } +) + + +@dataclass(frozen=True, slots=True) +class Settings: + input_wav: Path + output_wav: Path + base_url: str + model: str + target_language: str + trailing_silence_seconds: float + api_key: str + + +def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None: + sys.stdout.write(f"{message}{end}") + if flush: + sys.stdout.flush() + + +def write_stderr(message: str) -> None: + sys.stderr.write(f"{message}\n") + + +def parse_args(argv: Sequence[str] | None = None) -> Settings | str: + parser = argparse.ArgumentParser( + description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio", + ) + parser.add_argument("input_wav", type=Path) + parser.add_argument("--output", type=Path, default=Path("translated.wav")) + parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000")) + parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate")) + parser.add_argument("--target-language", default="fr") + parser.add_argument("--trailing-silence", type=float, default=1.5) + parsed = parser.parse_args(argv) + api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY") + if not api_key: + return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script" + if parsed.trailing_silence < 0: + return "--trailing-silence must be zero or greater" + return Settings( + input_wav=parsed.input_wav, + output_wav=parsed.output, + base_url=parsed.base_url, + model=parsed.model, + target_language=parsed.target_language, + trailing_silence_seconds=parsed.trailing_silence, + api_key=api_key, + ) + + +def translation_url(base_url: str, model: str) -> str | None: + parsed = urlsplit(base_url.rstrip("/")) + scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme) + if not scheme or not parsed.netloc: + return None + base_path = parsed.path.rstrip("/") + realtime_path = ( + f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations" + ) + return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), "")) + + +def read_pcm16_wav(path: Path) -> bytes | str: + try: + with wave.open(str(path), "rb") as source: + actual_format = ( + source.getnchannels(), + source.getsampwidth(), + source.getframerate(), + source.getcomptype(), + ) + expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE") + if actual_format != expected_format: + return ( + f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received " + f"channels={actual_format[0]}, sample_width={actual_format[1]}, " + f"sample_rate={actual_format[2]}, compression={actual_format[3]}" + ) + return source.readframes(source.getnframes()) + except (OSError, EOFError, wave.Error) as exc: + return f"Unable to read {path}: {exc}" + + +def audio_chunks(audio: bytes) -> Iterator[bytes]: + return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES)) + + +def audio_message(audio: bytes) -> str: + return json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(audio).decode("ascii"), + } + ) + + +async def configure_session(connection: ClientConnection, target_language: str) -> str | None: + await connection.send( + json.dumps( + { + "type": "session.update", + "session": {"audio": {"output": {"language": target_language}}}, + } + ) + ) + while True: + raw_event = await asyncio.wait_for(connection.recv(), timeout=20) + event = json.loads(raw_event) + event_type = event.get("type") + if event_type == "session.created": + write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}") + if event_type == "session.updated": + return None + if event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + + +async def send_audio( + connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event +) -> None: + silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + try: + for chunk in audio_chunks(pcm + silence): + await connection.send(audio_message(chunk)) + await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + finally: + finished.set() + + +async def receive_translation( + connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event +) -> str | None: + audio_received = asyncio.Event() + try: + with wave.open(str(output_path), "wb") as output: + output.setnchannels(CHANNELS) + output.setsampwidth(SAMPLE_WIDTH) + output.setframerate(SAMPLE_RATE) + write_stdout("Translation: ", end="", flush=True) + while True: + timeout = OUTPUT_IDLE_TIMEOUT_SECONDS if sender_finished.is_set() else INITIAL_RESPONSE_TIMEOUT_SECONDS + try: + raw_event = await asyncio.wait_for(connection.recv(), timeout=timeout) + except TimeoutError: + if sender_finished.is_set() and audio_received.is_set(): + write_stdout() + return None + return "The translation stream ended without translated audio" + event = json.loads(raw_event) + event_type = event.get("type") + if event_type in AUDIO_EVENT_TYPES: + output.writeframes(base64.b64decode(event.get("delta", ""), validate=True)) + audio_received.set() + elif event_type in TRANSCRIPT_EVENT_TYPES: + write_stdout(event.get("delta", event.get("text", "")), end="", flush=True) + elif event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + except (OSError, wave.Error) as exc: + return f"Unable to write {output_path}: {exc}" + + +async def translate(settings: Settings, pcm: bytes) -> str | None: + url = translation_url(settings.base_url, settings.model) + if not url: + return f"Invalid --base-url: {settings.base_url}" + sender_finished = asyncio.Event() + try: + async with websockets.connect( + url, + additional_headers={"Authorization": f"Bearer {settings.api_key}"}, + proxy=None, + open_timeout=20, + close_timeout=5, + ) as connection: + configuration_error = await configure_session(connection, settings.target_language) + if configuration_error: + return configuration_error + async with asyncio.TaskGroup() as tasks: + receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished)) + tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished)) + return receiver.result() + except Exception as exc: + return f"Translation failed: {type(exc).__name__}: {exc}" + + +def main(argv: Sequence[str] | None = None) -> int: + settings = parse_args(argv) + if isinstance(settings, str): + write_stderr(settings) + return 2 + pcm = read_pcm16_wav(settings.input_wav) + if isinstance(pcm, str): + write_stderr(pcm) + return 2 + error = asyncio.run(translate(settings, pcm)) + if error: + write_stderr(error) + return 1 + write_stdout(f"Translated audio: {settings.output_wav}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/litellm/__init__.py b/litellm/__init__.py index 1784deeb797..6e241a2baa1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -9,7 +9,7 @@ import threading import os import datetime -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from typing import ( Any, Callable, @@ -53,6 +53,7 @@ from litellm.constants import ( OPENAI_CHAT_COMPLETION_PARAMS as _openai_completion_params, # backwards compatibility OPENAI_FINISH_REASONS, OPENAI_FINISH_REASONS as _openai_finish_reasons, # backwards compatibility + OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS, openai_compatible_endpoints, openai_compatible_providers, openai_text_completion_compatible_providers, @@ -1034,6 +1035,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None): _populate_provider_model_sets(model_cost) +open_ai_chat_completion_models.update(OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS) # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary # this is maintained for Exception Mapping @@ -1512,7 +1514,9 @@ from .realtime_api.main import ( _arealtime, acreate_realtime_client_secret, acreate_realtime_transcription_session, + acreate_realtime_translation_client_secret, arealtime_calls, + arealtime_translation_calls, ) from .responses.main import _aresponses_websocket @@ -1745,6 +1749,9 @@ if TYPE_CHECKING: from .llms.vertex_ai.rerank.transformation import ( VertexAIRerankConfig as VertexAIRerankConfig, ) + from .llms.together_ai.chat.transformation import ( + TogetherAIChatConfig as TogetherAIChatConfig, + ) from .llms.fireworks_ai.rerank.transformation import ( FireworksAIRerankConfig as FireworksAIRerankConfig, ) @@ -1790,9 +1797,6 @@ if TYPE_CHECKING: BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig, ) from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig - from .llms.together_ai.chat.transformation import ( - TogetherAIChatConfig as TogetherAIChatConfig, - ) from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig as VertexGeminiConfig, @@ -1951,6 +1955,9 @@ if TYPE_CHECKING: from .llms.xai.responses.transformation import ( XAIResponsesAPIConfig as XAIResponsesAPIConfig, ) + from .llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig as VertexAIInteractionsConfig, + ) from .llms.litellm_proxy.responses.transformation import ( LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, ) @@ -1978,9 +1985,6 @@ if TYPE_CHECKING: from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) - from .llms.vertex_ai.interactions.transformation import ( - VertexAIInteractionsConfig as VertexAIInteractionsConfig, - ) from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config, diff --git a/litellm/constants.py b/litellm/constants.py index c0fe0491775..bc2ba343aa3 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -943,10 +943,44 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "language", + "languages", + "keywords", "response_format", + "stream", "timestamp_granularities", ] +OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS: Final = frozenset( + { + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-translate", + "gpt-realtime-whisper", + "gpt-transcribe", + "gpt-live-transcribe", + } +) + +AZURE_GA_REALTIME_MODELS: Final = frozenset( + { + "gpt-realtime-2", + "gpt-realtime-2-2026-05-06", + "gpt-realtime-2.1", + "gpt-realtime-2.1-2026-07-07", + "gpt-realtime-2.1-mini", + "gpt-realtime-2.1-mini-2026-07-07", + "gpt-realtime-translate", + "gpt-realtime-translate-2026-05-06", + "gpt-realtime-translate-2026-05-07", + "gpt-realtime-whisper", + "gpt-realtime-whisper-2026-05-06", + "gpt-realtime-whisper-2026-05-07", + "gpt-transcribe", + "gpt-live-transcribe", + } +) + OPENAI_EMBEDDING_PARAMS: Final = ["dimensions", "encoding_format", "user"] DEFAULT_EMBEDDING_PARAM_VALUES: Final = { diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 550bbfcde98..493924320be 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1229,6 +1229,24 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage: prompt_tokens_details=_decisions_prompt_tokens_details(usage), ) +def _get_transcription_usage_duration(completion_response: object) -> float | None: + usage_object: Final = ( + completion_response.get("usage") + if isinstance(completion_response, dict) + else getattr(completion_response, "usage", None) + ) + usage_type: Final = ( + usage_object.get("type") if isinstance(usage_object, dict) else getattr(usage_object, "type", None) + ) + if usage_type != "duration": + return None + seconds: Final = ( + usage_object.get("seconds") if isinstance(usage_object, dict) else getattr(usage_object, "seconds", None) + ) + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds < 0: + return None + return float(seconds) + def _extraction_usage(usage: ExtractionUsage) -> Usage: return Usage( @@ -1853,15 +1871,14 @@ def completion_cost( # the response attribute (for verbose_json responses that # naturally include duration from the provider). _hidden = getattr(completion_response, "_hidden_params", {}) or {} - audio_transcription_file_duration = _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), - ) or ( - completion_response.usage.seconds - if isinstance(completion_response, TranscriptionResponse) - and isinstance(completion_response.usage, TranscriptionUsageDurationObject) - and _bills_provider_transcription_seconds(requested_model, custom_llm_provider) - else 0.0 + provider_duration = _get_transcription_usage_duration(completion_response) + audio_transcription_file_duration = ( + provider_duration + if provider_duration is not None + else _hidden.get( + "audio_transcription_duration", + getattr(completion_response, "duration", 0.0), + ) ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): @@ -3253,6 +3270,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor): _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" +_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed" class _LiveResponsePayload(BaseModel): @@ -3519,6 +3537,11 @@ def handle_realtime_stream_cost_calculation( if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results) else 0.0 ) + translation_cost: Final = handle_realtime_translation_cost_calculation( + results=results, + custom_llm_provider=custom_llm_provider, + litellm_model_name=litellm_model_name, + ) live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"}) live_session_started: Final = any(result["type"] == "session.started" for result in results) live_session_closed: Final = any(result["type"] == "session.closed" for result in results) @@ -3575,12 +3598,13 @@ def handle_realtime_stream_cost_calculation( input_cost: Final = input_cost_per_token + live_response_input_cost output_cost: Final = output_cost_per_token + live_response_output_cost total_cost: Final = ( - input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost + input_cost + output_cost + transcription_cost + translation_cost + live_voice_cost + audio_duration_cost + text_message_cost ) additional_costs: Final = tuple( (cost_name, cost) for cost_name, cost in ( ("transcription_cost", transcription_cost), + ("translation_cost", translation_cost), ("live_voice_cost", live_voice_cost), ("audio_duration_cost", audio_duration_cost), ("text_message_cost", text_message_cost), @@ -3601,6 +3625,33 @@ def handle_realtime_stream_cost_calculation( return total_cost +def handle_realtime_translation_cost_calculation( + results: OpenAIRealtimeStreamList, + custom_llm_provider: str, + litellm_model_name: str, +) -> float: + output_seconds = 0.0 # rebind-ok: duration is accumulated across translation close events + for result in results: + if result.get("type") != _TRANSLATION_CLOSED_EVENT_TYPE: + continue + usage = result.get("usage") + if isinstance(usage, dict) and isinstance(usage.get("output_seconds"), (int, float)): + output_seconds += float(usage["output_seconds"]) + if output_seconds <= 0: + return 0.0 + try: + model_info: Final = litellm.get_model_info( + model=litellm_model_name, + custom_llm_provider=custom_llm_provider, + ) + except Exception: # noqa: BLE001 # unknown model metadata should yield zero translation cost + return 0.0 + output_cost_per_second: Final = model_info.get("output_cost_per_second") + if not isinstance(output_cost_per_second, (int, float)): + return 0.0 + return output_seconds * output_cost_per_second + + def handle_realtime_transcription_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py new file mode 100644 index 00000000000..5ae246f3468 --- /dev/null +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import datetime +import traceback +from collections.abc import AsyncIterator, Iterator +from typing import Final, Protocol + +from openai import AsyncStream, Stream +from openai.types.audio import ( + TranscriptionStreamEvent, + TranscriptionTextDeltaEvent, + TranscriptionTextDoneEvent, +) + +from litellm.types.utils import TranscriptionResponse + + +class TranscriptionStreamLogging(Protocol): + def success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def handle_sync_success_callbacks_for_async_calls( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + +class _TranscriptionEventCollector: + def __init__(self, duration: float | None) -> None: + self.duration = duration + self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event + self.done_event: TranscriptionTextDoneEvent | None = None + + def add(self, event: TranscriptionStreamEvent) -> None: + if isinstance(event, TranscriptionTextDeltaEvent): + self.text_deltas.append(event.delta) + elif isinstance(event, TranscriptionTextDoneEvent): + self.done_event = event + + def response(self) -> TranscriptionResponse: + done_event: Final = self.done_event + done_languages: Final = getattr(done_event, "languages", None) if done_event is not None else None + response: Final = TranscriptionResponse( + text=done_event.text if done_event is not None else "".join(self.text_deltas), + usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, + languages=( + [ # mutable-ok: the response model requires a concrete serialized language list + language.model_dump() for language in done_languages + ] + if done_languages is not None + else None + ), + ) + if self.duration is not None: + response.set_audio_transcription_duration(self.duration) + return response + + +class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): + def __init__( + self, + stream: Stream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + ) -> None: + self.__dict__.update(stream.__dict__) + self._logging_obj = logging_obj + self._start_time = start_time + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + def _logging_iterator( + self, source_iterator: Iterator[TranscriptionStreamEvent] + ) -> Iterator[TranscriptionStreamEvent]: + try: + for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(exception, traceback.format_exc(), self._start_time, end_time) + raise + finally: + self._finalize() + + def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + self._logging_obj.success_handler( + self._collector.response(), + self._start_time, + datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract + ) + + def close(self) -> None: + try: + super().close() + finally: + self._finalize() + + +class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): + def __init__( + self, + stream: AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + ) -> None: + self.__dict__.update(stream.__dict__) + self._logging_obj = logging_obj + self._start_time = start_time + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + async def _logging_iterator( + self, source_iterator: AsyncIterator[TranscriptionStreamEvent] + ) -> AsyncIterator[TranscriptionStreamEvent]: + try: + async for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(exception, traceback.format_exc(), self._start_time, end_time) + await self._logging_obj.async_failure_handler( + exception, + traceback.format_exc(), + self._start_time, + end_time, + ) + raise + finally: + await self._finalize() + + async def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + response: Final = self._collector.response() + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.handle_sync_success_callbacks_for_async_calls( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + await self._logging_obj.async_success_handler( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + + async def close(self) -> None: + try: + await super().close() + finally: + await self._finalize() + + +def wrap_transcription_stream( + stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, +) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream: + if isinstance(stream, AsyncStream): + return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time) + return LoggingTranscriptionStream(stream, logging_obj, start_time) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a874459798d..b0e5e221882 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -999,6 +999,11 @@ def calculate_cache_writing_cost( class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int cache_hit_audio_tokens: ReadOnly[int] + + cached_text_tokens: ReadOnly[int] + cached_audio_tokens: ReadOnly[int] + cached_image_tokens: ReadOnly[int] + has_cached_tokens_details: ReadOnly[bool] cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None text_tokens: int @@ -1085,6 +1090,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, cache_hit_audio_tokens=cached_audio_tokens, + cached_text_tokens=cached_text_tokens, + cached_audio_tokens=cached_audio_tokens, + cached_image_tokens=cached_image_tokens, + has_cached_tokens_details=cached_tokens_details is not None, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, @@ -1193,15 +1202,11 @@ def _calculate_input_cost( prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing - cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] - audio_cache_read_rate: Final = get_cost_per_unit( - model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost - prompt_cost += float(cache_hit_audio_tokens) * ( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost + prompt_cost += _calculate_cache_read_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=model_info, + cache_read_cost=cache_read_cost, + service_tier=service_tier, ) ### AUDIO COST @@ -1290,6 +1295,38 @@ def _calculate_input_cost( return prompt_cost +def _calculate_cache_read_cost( + prompt_tokens_details: PromptTokensDetailsResult, + model_info: ModelInfo, + cache_read_cost: float, + service_tier: str | None, +) -> float: + cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"] + cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"] + cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"] + classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens + unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0) + total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost + float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost + ) + + if cached_audio_tokens: + cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier) + cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached audio contributes to cache-read cost + float(cached_audio_tokens) * float(cached_audio_cost or 0.0) + ) + + if cached_image_tokens: + cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier) + cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached images contribute to cache-read cost + float(cached_image_tokens) * float(cached_image_cost or 0.0) + ) + + return total_cost + + def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: """ Resolve the per-model regional-processing uplift multiplier for a given @@ -1461,6 +1498,10 @@ def generic_cost_per_token( prompt_tokens_details = PromptTokensDetailsResult( cache_hit_tokens=0, cache_hit_audio_tokens=0, + cached_text_tokens=0, + cached_audio_tokens=0, + cached_image_tokens=0, + has_cached_tokens_details=False, cache_creation_tokens=0, cache_creation_token_details=None, text_tokens=usage.prompt_tokens, @@ -1642,6 +1683,7 @@ class BilledTokenRates: cache_creation_input_token_cost: float cache_creation_input_token_cost_above_1hr: float output_cost_per_reasoning_token: float + cache_read_input_image_token_cost: float | None = None def scaled(self, multiplier: float) -> "BilledTokenRates": if multiplier == 1.0: @@ -1654,6 +1696,11 @@ class BilledTokenRates: cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, + cache_read_input_image_token_cost=( + self.cache_read_input_image_token_cost * multiplier + if self.cache_read_input_image_token_cost is not None + else None + ), ) @@ -1768,6 +1815,11 @@ def _cost_map_billed_rates( cache_creation_input_token_cost=cache_creation_cost_rate, cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, output_cost_per_reasoning_token=reasoning_rate, + cache_read_input_image_token_cost=_get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier), + None, + ), ).scaled(multiplier) @@ -1848,6 +1900,12 @@ def get_token_type_cost_breakdown( cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( usage ) + cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"] + image_cache_read_rate: Final = ( + rates.cache_read_input_image_token_cost + if rates.cache_read_input_image_token_cost is not None + else rates.cache_read_input_token_cost + ) cache_creation_cost: Final = ( float(cache_creation_tokens) * rates.cache_creation_input_token_cost if custom_cost_per_token is not None @@ -1861,8 +1919,9 @@ def get_token_type_cost_breakdown( return TokenTypeCostBreakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, cache_read_cost=( - float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost + float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + + float(cached_image_tokens) * image_cache_read_rate ), cache_creation_cost=cache_creation_cost, rates=rates, diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8b94c34942c..8de9c9810fd 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -44,6 +44,7 @@ from litellm.types.utils import ( StreamingChoices, TextChoices, TextCompletionResponse, + TranscriptionDetectedLanguage, TranscriptionResponse, TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, @@ -899,9 +900,11 @@ def convert_to_model_response_object( model_response_object.data = response_object["data"] if "usage" in response_object and response_object["usage"] is not None: - model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) - model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) - model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) + embedding_usage: Final = model_response_object.usage or Usage() + embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) + embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) + embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0) + model_response_object.usage = embedding_usage if start_time is not None and end_time is not None: model_response_object._response_ms = ( @@ -944,6 +947,12 @@ def convert_to_model_response_object( if key in response_object: setattr(model_response_object, key, response_object[key]) + if "languages" in response_object and response_object["languages"] is not None: + transcription_response: Final = model_response_object + transcription_response.languages = tuple( + TranscriptionDetectedLanguage.model_validate(language) for language in response_object["languages"] + ) + if "usage" in response_object and response_object["usage"] is not None: tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 927caec3350..824829cd926 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,4 +1,5 @@ import asyncio +import base64 import json import traceback from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence @@ -27,6 +28,8 @@ from litellm.types.llms.openai import ( OpenAIRealtimeResponseDelta, OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamSessionEvents, + OpenAIRealtimeTranslationClosedEvent, + OpenAIRealtimeTranslationDurationUsage, ) from litellm.types.realtime import ALL_DELTA_TYPES @@ -242,6 +245,7 @@ class RealTimeStreaming: event_normalizer: RealtimeEventNormalizer | None = None, usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + translation_session: bool = False, ): self.websocket: _ClientWebSocket = websocket self._replay_scope: Final = client_websocket_scope(websocket) @@ -254,6 +258,10 @@ class RealTimeStreaming: self.input_messages: list[dict[str, str]] = [] self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] + self._is_translation_session = translation_session + self._translation_output_audio_bytes = 0 + self._translation_output_bytes_per_second = 48000.0 + self._translation_usage_finalized = False # Detect whether the client is explicitly opting into the beta protocol. self._client_wants_beta = self.detect_beta_header(websocket) @@ -304,6 +312,7 @@ class RealTimeStreaming: # their input_audio_transcription.completed usage drives duration-based cost. self._force_transcription_model = force_transcription_model self._is_transcription_session: bool = force_transcription_model is not None + self._bound_nested_transcription_model: str | None = None # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer). self._event_normalizer = event_normalizer self._gate_turn_detection: Final[_TurnDetectionGate] = ( @@ -532,6 +541,7 @@ class RealTimeStreaming: async def log_messages(self): """Log messages in list""" + self._finalize_translation_usage() if self.logging_obj: if self._usage_meter is not None: self.logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] = self._usage_meter.snapshot() @@ -548,6 +558,60 @@ class RealTimeStreaming: ) self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True + def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None: + if not self._is_translation_session: + return + self._capture_translation_output_format(event_obj) + if event_obj.get("type") != "session.output_audio.delta": + return + delta: Final = event_obj.get("delta") + if not isinstance(delta, str): + return + try: + decoded: Final = base64.b64decode(delta, validate=True) + except (ValueError, TypeError): + return + self._translation_output_audio_bytes += len(decoded) + + def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: + session: Final = event_obj.get("session") + if not isinstance(session, dict): + return + audio: Final = session.get("audio") + output: Final = audio.get("output") if isinstance(audio, dict) else None + audio_format: Final = output.get("format") if isinstance(output, dict) else None + if isinstance(audio_format, str): + if audio_format in ("g711_ulaw", "g711_alaw"): + self._translation_output_bytes_per_second = 8000.0 + return + if not isinstance(audio_format, dict): + return + format_type: Final = audio_format.get("type") + rate: Final = audio_format.get("rate") + if not isinstance(rate, (int, float)) or rate <= 0: + return + if format_type == "audio/pcm": + self._translation_output_bytes_per_second = float(rate) * 2 + elif format_type in ("audio/pcmu", "audio/pcma"): + self._translation_output_bytes_per_second = float(rate) + + def _finalize_translation_usage(self) -> None: + if self._translation_usage_finalized: + return + for event in self.messages: + if event.get("type") != "session.closed": + continue + event_usage = event.get("usage") # rebind-ok: each close event carries independent usage + if isinstance(event_usage, dict) and isinstance(event_usage.get("output_seconds"), (int, float)): + self._translation_usage_finalized = True + return + if self._translation_output_audio_bytes == 0: + return + output_seconds: Final = self._translation_output_audio_bytes / self._translation_output_bytes_per_second + synthetic_usage: Final = OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) + self._translation_usage_finalized = True + async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. @@ -560,7 +624,7 @@ class RealTimeStreaming: backend, False if the provider transformation produced no output and the message was effectively dropped. """ - message = self._enforce_transcription_session_model(message) + message = await self._apply_nested_transcription_model_policy(message) if self.provider_config: transformed: Final = self.provider_config.transform_realtime_request( message, self.model, self.session_configuration_request @@ -606,6 +670,90 @@ class RealTimeStreaming: self._usage_meter.observe_client_event(sent_event) return True + async def _apply_nested_transcription_model_policy(self, message: str) -> str: + if self._force_transcription_model is not None: + return self._enforce_transcription_session_model(message) + if self._is_translation_session: + return await self._enforce_translation_nested_transcription_model(message) + return message + + def _session_update_message_obj(self, message: str) -> Mapping[str, object] | None: + try: + message_obj: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return None + if message_obj.get("type") not in ( + "session.update", + "transcription_session.update", + ): + return None + return message_obj + + def _nested_transcription_models_from_session( + self, + session: Mapping[str, object], + ) -> tuple[str, ...]: + audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None + nested_model: Final = self._transcription_model_value(nested_transcription) + flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription")) + return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None)) + + def _transcription_model_value(self, transcription_config: object) -> str | None: + if not isinstance(transcription_config, dict): + return None + model: Final = transcription_config.get("model") + if isinstance(model, str) and model: + return model + return None + + def _rewrite_session_update_transcription_model(self, message: str, authorized_model: str) -> str: + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: + return message + session: Final = message_obj.get("session") + if not isinstance(session, dict): + return message + + transcription: Final = session.get("input_audio_transcription") + rewrite_flat: Final = isinstance(transcription, dict) and transcription.get("model") != authorized_model + if isinstance(transcription, dict) and rewrite_flat: + session["input_audio_transcription"] = { + **transcription, + "model": authorized_model, + } + + audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None + rewrite_nested: Final = ( + isinstance(audio, dict) + and isinstance(audio_input, dict) + and isinstance(nested_transcription, dict) + and nested_transcription.get("model") != authorized_model + ) + if ( + isinstance(audio, dict) + and isinstance(audio_input, dict) + and isinstance(nested_transcription, dict) + and rewrite_nested + ): + session["audio"] = { + **audio, + "input": { + **audio_input, + "transcription": { + **nested_transcription, + "model": authorized_model, + }, + }, + } + + if not rewrite_flat and not rewrite_nested: + return message + return json.dumps(message_obj) + def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -623,56 +771,49 @@ class RealTimeStreaming: if self._force_transcription_model is None: return message - try: - message_obj: Final = _decode_json_object(message) - except (json.JSONDecodeError, TypeError): + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message + session: Final = message_obj.get("session") + if isinstance(session, dict) and session.get("type") == "transcription": + self._is_transcription_session = True + return self._rewrite_session_update_transcription_model(message, self._force_transcription_model) - if message_obj.get("type") not in ( - "session.update", - "transcription_session.update", - ): + async def _enforce_translation_nested_transcription_model(self, message: str) -> str: + if self._bound_nested_transcription_model is not None: + return self._rewrite_session_update_transcription_model(message, self._bound_nested_transcription_model) + + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message - session: Final = message_obj.get("session") if not isinstance(session, dict): return message - - if session.get("type") == "transcription": - self._is_transcription_session = True - - authorized_model: Final = self._force_transcription_model - changed = False - - transcription: Final = session.get("input_audio_transcription") - if isinstance(transcription, dict) and transcription.get("model") != authorized_model: - session["input_audio_transcription"] = { - **transcription, - "model": authorized_model, - } - changed = True - - audio: Final = session.get("audio") - if isinstance(audio, dict): - audio_input: Final = audio.get("input") - if isinstance(audio_input, dict): - nested_transcription: Final = audio_input.get("transcription") - if isinstance(nested_transcription, dict) and nested_transcription.get("model") != authorized_model: - session["audio"] = { - **audio, - "input": { - **audio_input, - "transcription": { - **nested_transcription, - "model": authorized_model, - }, - }, - } - changed = True - - if not changed: + nested_models: Final = self._nested_transcription_models_from_session(session) + if not nested_models: return message - return json.dumps(message_obj) + + valid_token: Final = self.user_api_key_dict + if valid_token is None: + return message + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import can_key_call_resolved_model + from litellm.proxy.proxy_server import llm_model_list, llm_router + + if not isinstance(valid_token, UserAPIKeyAuth): + return message + + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + bound_model: Final = nested_models[0] + self._bound_nested_transcription_model = bound_model + return self._rewrite_session_update_transcription_model(message, bound_model) def _uses_deferred_backend_setup(self) -> bool: """True when setup is deferred until the client's first session.update.""" @@ -1094,7 +1235,10 @@ class RealTimeStreaming: async def _handle_provider_config_message(self, raw_response: str) -> None: """Process a backend message when a provider_config is set (transformed path).""" - returned_object: Final = self.provider_config.transform_realtime_response( + provider_config: Final = self.provider_config + if provider_config is None: + raise RuntimeError("Provider response handling requires a provider configuration") + returned_object: Final = provider_config.transform_realtime_response( raw_response, self.model, self.logging_obj, @@ -1257,6 +1401,7 @@ class RealTimeStreaming: verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event) continue + self._capture_translation_output_audio(event) if await self._handle_raw_backend_message(event, raw_response): continue @@ -1645,6 +1790,8 @@ class RealTimeStreaming: session = client_event.get("session", {}) if isinstance(session, dict): session = self._remap_beta_session_to_ga(session) + if self._is_translation_session: + session.pop("type", None) msg_obj["session"] = session message = json.dumps(msg_obj) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 99ebd2875a7..f8e4875ab34 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,7 +1,7 @@ from collections.abc import Coroutine from typing import TYPE_CHECKING, Any, Final -from openai import AsyncAzureOpenAI, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from pydantic import BaseModel from litellm._uuid import uuid @@ -49,6 +49,7 @@ class AzureAudioTranscription(AzureChatCompletion): timeout=timeout, api_key=api_key, api_base=api_base, + api_version=api_version, client=client, max_retries=max_retries, logging_obj=logging_obj, @@ -66,7 +67,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(azure_client, AzureOpenAI): + if not isinstance(azure_client, (AzureOpenAI, OpenAI)): raise AzureOpenAIError( status_code=500, message="azure_client is not an instance of AzureOpenAI", @@ -85,10 +86,13 @@ class AzureAudioTranscription(AzureChatCompletion): ) response: Final = azure_client.audio.transcriptions.create( - **data, + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) + if data.get("stream") is True: + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -137,7 +141,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(async_azure_client, AsyncAzureOpenAI): + if not isinstance(async_azure_client, (AsyncAzureOpenAI, AsyncOpenAI)): raise AzureOpenAIError( status_code=500, message="async_azure_client is not an instance of AsyncAzureOpenAI", @@ -155,8 +159,15 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) + if data.get("stream") is True: + return await async_azure_client.audio.transcriptions.create( + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, + ) + raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( - **data, timeout=timeout + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, ) headers: Final = dict(raw_response.headers) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 3d4576fd7ac..4e93047b8a0 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -92,6 +92,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version: str | None, realtime_protocol: str | None = None, query_params: RealtimeQueryParams | None = None, + realtime_mode: str = "realtime", ) -> str: """ Construct Azure realtime WebSocket URL. @@ -123,18 +124,26 @@ class AzureOpenAIRealtime(AzureChatCompletion): ) intent: Final = (query_params or {}).get("intent") - if _is_ga: - path = "/openai/v1/realtime" - query_parts = [] - if intent != "transcription" and (query_params is None or "model" in query_params): - query_parts.append(urlencode({"model": model})) - else: - # Default to beta path for backwards compatibility - path = "/openai/realtime" - query_parts = [urlencode({"api-version": api_version, "deployment": model})] + path: Final = ( + "/openai/v1/realtime/translations" + if realtime_mode == "translation" + else "/openai/v1/realtime" + if _is_ga + else "/openai/realtime" + ) + base_query_parts: Final = ( + (urlencode((("model", model),)),) + if realtime_mode == "translation" + else ( + (urlencode((("model", model),)),) + if intent != "transcription" and (query_params is None or "model" in query_params) + else () + ) + if _is_ga + else (urlencode((("api-version", api_version), ("deployment", model))),) + ) - if intent: - query_parts.append(urlencode({"intent": intent})) + query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts qs: Final = "&".join(query_parts) endpoint_url: Final = join_api_base(api_base, path) @@ -165,6 +174,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", ): import websockets from websockets.asyncio.client import ClientConnection @@ -181,6 +191,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, ) auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token) @@ -204,6 +215,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): force_transcription_model=( model if (query_params or {}).get("intent") == "transcription" else None ), + translation_session=realtime_mode == "translation", ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 545c3348182..ea2c2990182 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -3,6 +3,7 @@ from typing import Final import litellm +from litellm.constants import AZURE_GA_REALTIME_MODELS from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str @@ -15,6 +16,10 @@ def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str: class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): + @staticmethod + def _uses_ga_api(model: str, api_version: str | None) -> bool: + return api_version in ("preview", "latest", "v1") or model in AZURE_GA_REALTIME_MODELS + def get_api_base(self, api_base: str | None, **kwargs: object) -> str: return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or "" @@ -30,20 +35,32 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): model: str, api_key: str | None = None, ) -> dict: - return { + validated_headers: Final = { # mutable-ok: provider authentication headers are extended before dispatch **headers, - "api-key": api_key or "", "Content-Type": "application/json", } + if api_key: + validated_headers["api-key"] = api_key + return validated_headers def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: return _azure_realtime_v1_url(self.get_api_base(api_base), "calls") def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base) + if self._uses_ga_api(model, api_version): + return f"{base.rstrip('/')}/openai/v1/realtime/transcription_sessions" version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/client_secrets") + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/calls") + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]: return { "Authorization": f"Bearer {ephemeral_key}", diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index a02054ec8cb..18363d288e4 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -64,6 +64,12 @@ class BaseRealtimeHTTPConfig(ABC): base = base.removesuffix("/v1") return join_api_base(base, "v1/realtime/transcription_sessions") + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/client_secrets" + @abstractmethod def validate_environment( self, @@ -86,6 +92,10 @@ class BaseRealtimeHTTPConfig(ABC): """Return the full URL for POST /realtime/calls (SDP exchange).""" return join_api_base(api_base or "", "v1/realtime/calls") + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/calls" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: """ Build headers for the realtime_calls POST. diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a7ca0d5f49e..b6233899abd 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -27,10 +27,13 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse import httpx from httpx import USE_CLIENT_DEFAULT from httpx._types import FileContent +from openai import AsyncOpenAI from openai.types.file_deleted import FileDeleted from pydantic import TypeAdapter from typing_extensions import ReadOnly +from openai.types.realtime import RealtimeSessionCreateRequestParam + import litellm import litellm.litellm_core_utils import litellm.types @@ -6846,9 +6849,10 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, + use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/client_secrets to upstream provider. @@ -6856,6 +6860,52 @@ class BaseLLMHTTPHandler: Uses provider_config (BaseRealtimeHTTPConfig) for URL construction and header auth when available; falls back to the legacy OpenAI-style defaults. """ + if use_openai_sdk: + trimmed_api_base: Final = api_base.rstrip("/") + normalized_api_base: Final = ( + trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" + ) + owns_client: Final = not isinstance(client, AsyncOpenAI) + openai_client: Final = ( + client + if isinstance(client, AsyncOpenAI) + else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) + ) + logging_obj.pre_call( + input=request_data, + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "complete_input_dict": request_data, + "api_base": normalized_api_base, + }, + ) + try: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + key: str(value) # mutable-ok: SDK headers are materialized as a concrete string mapping + for key, value in (extra_headers or {}).items() # mutable-ok: SDK requires concrete headers + }, + ) + raw_response: Final = await configured_client.post( + "/realtime/client_secrets", + cast_to=httpx.Response, + body=request_data, + ) + response_headers: Final = { # mutable-ok: httpx requires a concrete response-header mapping + key: value # mutable-ok: transport headers are materialized after filtering + for key, value in raw_response.headers.items() # mutable-ok: transport headers are materialized + if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") + } + return httpx.Response( + status_code=raw_response.status_code, + headers=response_headers, + content=raw_response.content, + request=httpx.Request("POST", f"{normalized_api_base}/realtime/client_secrets"), + ) + finally: + if owns_client: + await openai_client.close() return await self._async_realtime_session_post( endpoint="client_secrets", api_base=api_base, @@ -6879,8 +6929,8 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, ) -> httpx.Response: """Forward POST /v1/realtime/transcription_sessions to upstream provider.""" @@ -6898,9 +6948,8 @@ class BaseLLMHTTPHandler: api_version=api_version, ) - async def _async_realtime_session_post( + async def async_realtime_translation_client_secret_handler( self, - endpoint: Literal["client_secrets", "transcription_sessions"], api_base: str, api_key: str, request_data: dict[str, object], @@ -6908,8 +6957,70 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + api_version: str | None = None, + use_openai_sdk: bool = False, + ) -> httpx.Response: + if use_openai_sdk: + normalized_api_base = api_base.rstrip("/") + if not normalized_api_base.endswith("/v1"): + normalized_api_base = f"{normalized_api_base}/v1" + owns_client: Final = not isinstance(client, AsyncOpenAI) + openai_client: Final = ( + client + if isinstance(client, AsyncOpenAI) + else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) + ) + logging_obj.pre_call( + input=request_data, + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "complete_input_dict": request_data, + "api_base": normalized_api_base, + }, + ) + try: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + key: str(value) for key, value in (extra_headers or {}).items() + }, + ) + return await configured_client.post( + "/realtime/translations/client_secrets", + cast_to=httpx.Response, + body=request_data, + ) + finally: + if owns_client: + await openai_client.close() + return await self._async_realtime_session_post( + endpoint="translation_client_secrets", + api_base=api_base, + api_key=api_key, + request_data=request_data, + logging_obj=logging_obj, + timeout=timeout, + provider_config=provider_config, + model=model, + extra_headers=extra_headers, + client=client, + api_version=api_version, + ) + + async def _async_realtime_session_post( + self, + endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"], + api_base: str, + api_key: str, + request_data: dict[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + provider_config: BaseRealtimeHTTPConfig | None = None, + model: str | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, ) -> httpx.Response: """ @@ -6931,13 +7042,20 @@ class BaseLLMHTTPHandler: url = provider_config.get_transcription_session_url( api_base=api_base, model=model or "", api_version=api_version ) + elif endpoint == "translation_client_secrets": + url = provider_config.get_translation_client_secret_url( + api_base=api_base, model=model or "", api_version=api_version + ) else: url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version) headers: dict[str, object] = provider_config.validate_environment( headers={}, model=model or "", api_key=api_key ) else: - url = join_api_base(api_base, f"v1/realtime/{endpoint}") + endpoint_path: Final = ( + "translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint + ) + url = join_api_base(api_base, f"v1/realtime/{endpoint_path}") headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", @@ -6971,6 +7089,80 @@ class BaseLLMHTTPHandler: ) raise + async def _async_realtime_calls_sdk( + self, + api_base: str, + openai_ephemeral_key: str, + sdp_text: str, + session_data: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + extra_headers: Mapping[str, object] | None, + client: object | None, + translation: bool, + ) -> httpx.Response: + normalized_api_base = api_base.rstrip("/") + if not normalized_api_base.endswith("/v1"): + normalized_api_base = f"{normalized_api_base}/v1" + owns_client: Final = not isinstance(client, AsyncOpenAI) + openai_client: Final = ( + client + if isinstance(client, AsyncOpenAI) + else AsyncOpenAI(api_key=openai_ephemeral_key, base_url=normalized_api_base, max_retries=0) + ) + logging_obj.pre_call( + input="realtime_sdp_offer", + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "api_base": normalized_api_base, + "session": session_data, + }, + ) + try: + if translation: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + "Content-Type": "application/sdp", + **{ # mutable-ok: caller headers are normalized into the SDK header mapping + key: str(value) for key, value in (extra_headers or {}).items() + }, + }, + ) + return await configured_client.post( + "/realtime/translations/calls", + cast_to=httpx.Response, + content=sdp_text.encode("utf-8"), + ) + realtime_session_data: Final = cast( # cast-ok: endpoint validation produced an OpenAI realtime session + RealtimeSessionCreateRequestParam, + session_data, + ) + sdk_extra_headers: Final = { # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + key: str(value) for key, value in (extra_headers or {}).items() + } + raw_response: Final = await openai_client.realtime.calls.with_raw_response.create( + sdp=sdp_text, + session=realtime_session_data, + extra_headers=sdk_extra_headers, + timeout=timeout, + ) + return httpx.Response( + status_code=raw_response.status_code, + headers=raw_response.headers, + content=raw_response.content, + request=httpx.Request("POST", f"{normalized_api_base}/realtime/calls"), + ) + finally: + if owns_client: + await openai_client.close() + + @staticmethod + def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: + if isinstance(client, AsyncHTTPHandler): + return client + return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI) + async def async_realtime_calls_handler( self, api_base: str, @@ -6980,10 +7172,12 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - session_config: dict[str, object] | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + session_config: Mapping[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, + translation: bool = False, + use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. @@ -6995,18 +7189,45 @@ class BaseLLMHTTPHandler: - sdp: the SDP offer (text) - session: JSON string with {"type": "realtime", "model": "...", ...} """ - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders.OPENAI, + session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally + **( + session_config or {} # mutable-ok: absent session configuration starts from an empty provider payload ) - else: - async_httpx_client = client + } + if "type" not in session_data: + session_data["type"] = "translation" if translation else "realtime" + if "model" not in session_data and model: + session_data["model"] = model + + sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + + if use_openai_sdk: + return await self._async_realtime_calls_sdk( + api_base=api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_text=sdp_text, + session_data=session_data, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=client, + translation=translation, + ) + + async_httpx_client: Final = self._get_realtime_async_http_client(client) if provider_config is not None: - url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) + url = ( + provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version) + if translation + else provider_config.get_realtime_calls_url( + api_base=api_base, model=model or "", api_version=api_version + ) + ) headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key) else: - url = join_api_base(api_base, "v1/realtime/calls") + path: Final = "translations/calls" if translation else "calls" + url = join_api_base(api_base, f"v1/realtime/{path}") headers = { "Authorization": f"Bearer {openai_ephemeral_key}", } @@ -7014,14 +7235,8 @@ class BaseLLMHTTPHandler: if extra_headers: headers.update(extra_headers) - # Build multipart form data: sdp + session JSON - session_data: Final = session_config or {} - if "type" not in session_data: - session_data["type"] = "realtime" - if "model" not in session_data and model: - session_data["model"] = model - - sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + if translation: + headers["Content-Type"] = "application/sdp" files: Final = { "sdp": (None, sdp_text, "text/plain"), @@ -7039,12 +7254,14 @@ class BaseLLMHTTPHandler: ) try: - return await async_httpx_client.post( - url=url, - headers=headers, - files=files, - timeout=timeout, - ) + if translation: + return await async_httpx_client.post( + url=url, + headers=headers, + content=sdp_text, + timeout=timeout, + ) + return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout) except Exception as e: if provider_config is not None: raise self._handle_error( diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 080fc3d24c9..ef63afa269b 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -5,8 +5,17 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ import ssl +from collections.abc import Mapping +from contextlib import AbstractAsyncContextManager +from types import TracebackType from typing import Any, Final, cast +from openai import AsyncOpenAI, omit +from openai.resources.realtime.realtime import ( + AsyncRealtimeConnection, + AsyncRealtimeConnectionManager, +) + from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams @@ -24,6 +33,49 @@ from ..common_utils import is_openai_backed_api_base from ..openai import OpenAIChatCompletion +class OpenAIRealtimeConnectionAdapter: + def __init__(self, connection: AsyncRealtimeConnection) -> None: + self._connection = connection + + async def send(self, message: str) -> None: + await self._connection.send_raw(message) + + async def recv(self, decode: bool = True) -> str | bytes: + message: Final = await self._connection.recv_bytes() + if decode: + return message.decode("utf-8") + return message + + async def close(self) -> None: + await self._connection.close() + + +class OpenAIRealtimeSDKConnectionManager: + def __init__( + self, + manager: AsyncRealtimeConnectionManager, + owned_client: AsyncOpenAI | None = None, + ) -> None: + self._manager = manager + self._owned_client = owned_client + + async def __aenter__(self) -> OpenAIRealtimeConnectionAdapter: + connection: Final = await self._manager.__aenter__() + return OpenAIRealtimeConnectionAdapter(connection) + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + await self._manager.__aexit__(exc_type, exc, traceback) + finally: + if self._owned_client is not None: + await self._owned_client.close() + + class OpenAIRealtime(OpenAIChatCompletion): """ Base handler for OpenAI-compatible realtime WebSocket connections. @@ -84,7 +136,7 @@ class OpenAIRealtime(OpenAIChatCompletion): return ssl_config - def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str: + def _construct_url(self, api_base: str, query_params: RealtimeQueryParams, realtime_mode: str = "realtime") -> str: """ Construct the backend websocket URL with the client's query parameters. @@ -101,7 +153,8 @@ class OpenAIRealtime(OpenAIChatCompletion): api_base = api_base.replace("http://", "ws://") url = URL(api_base) # Set the correct path - url = url.copy_with(path="/v1/realtime") + path: Final = "/v1/realtime/translations" if realtime_mode == "translation" else "/v1/realtime" + url = url.copy_with(path=path) upstream_params: Final = tuple( (key, value) for key, value in query_params.items() if not (drops_model and key == "model") ) @@ -128,6 +181,47 @@ class OpenAIRealtime(OpenAIChatCompletion): """Return a per-session usage meter, or None when billing needs no metering.""" return None + def _create_connection_manager( + self, + api_base: str, + api_key: str, + model: str, + query_params: RealtimeQueryParams, + headers: Mapping[str, str], + timeout: float | None, + realtime_mode: str, + ssl_config: object, + client: object | None, + url: str, + ) -> AbstractAsyncContextManager[object]: + import websockets + + if realtime_mode == "translation" or client is None: + return websockets.connect( + url, + additional_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=ssl_config, + ) + if not isinstance(client, AsyncOpenAI): + raise TypeError("client must be an AsyncOpenAI instance") + openai_client: Final = client + model_query: Final = query_params.get("model") + extra_query: Final = { # mutable-ok: OpenAI SDK accepts a mutable query-parameter mapping + key: value for key, value in query_params.items() if key != "model" + } + sdk_model: Final = omit if query_params.get("intent") == "transcription" else model_query or model + sdk_connection_manager: Final = openai_client.realtime.connect( + model=sdk_model, + extra_query=extra_query, + extra_headers=headers, + websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping + "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + }, + max_retries=0, + ) + return OpenAIRealtimeSDKConnectionManager(sdk_connection_manager) + async def async_realtime( self, model: str, @@ -140,6 +234,7 @@ class OpenAIRealtime(OpenAIChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", **kwargs: object, ): import websockets @@ -153,7 +248,7 @@ class OpenAIRealtime(OpenAIChatCompletion): # Use all query params if provided, else fallback to just model if query_params is None: query_params = {"model": model} - url: Final = self._construct_url(api_base, query_params) + url: Final = self._construct_url(api_base, query_params, realtime_mode=realtime_mode) try: # Get provider-specific SSL configuration @@ -178,15 +273,25 @@ class OpenAIRealtime(OpenAIChatCompletion): "complete_input_dict": {"query_params": query_params}, }, ) - async with websockets.connect( - url, - additional_headers=headers, - max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - ssl=ssl_config, - ) as backend_ws: + connection_manager: Final = self._create_connection_manager( + api_base=api_base, + api_key=api_key, + model=model, + query_params=query_params, + headers=headers, + timeout=timeout, + realtime_mode=realtime_mode, + ssl_config=ssl_config, + client=client, + url=url, + ) + + async with connection_manager as backend_ws: realtime_streaming: Final = RealTimeStreaming( websocket, - cast(ClientConnection, backend_ws), + cast( # cast-ok: both SDK and websockets adapters implement the streaming connection interface + ClientConnection, backend_ws + ), logging_obj, model=model, user_api_key_dict=user_api_key_dict, @@ -196,6 +301,8 @@ class OpenAIRealtime(OpenAIChatCompletion): ), event_normalizer=self._make_event_normalizer(), usage_meter=self._make_usage_meter(), + + translation_session=realtime_mode == "translation", ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py index 49d33582c6f..ab32ae6085e 100644 --- a/litellm/llms/openai/realtime/http_transformation.py +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -28,6 +28,18 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): base = base.removesuffix("/v1") return join_api_base(base, "v1/realtime/transcription_sessions") + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/client_secrets" + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/calls" + def validate_environment( self, headers: dict, diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index e7f8dd04516..81f3abd217a 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -14,7 +14,7 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): """ Get the supported OpenAI params for the `gpt-4o-transcribe` models """ - return [ + return [ # mutable-ok: base transcription interface requires a mutable supported-parameter list "language", "prompt", "response_format", @@ -37,3 +37,31 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): return AudioTranscriptionRequestData( data=data, ) + + +class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig): + def get_supported_openai_params( # mutable-ok: base transcription interface returns a mutable parameter list + self, model: str + ) -> list[OpenAIAudioTranscriptionOptionalParams]: + return [ + "prompt", + "response_format", + "keywords", + "languages", + "stream", + ] + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, # mutable-ok: base transformation interface supplies a mutable request payload + litellm_params: dict, # mutable-ok: base transformation interface supplies mutable provider parameters + ) -> AudioTranscriptionRequestData: + data: Final = { # mutable-ok: OpenAI SDK consumes this multipart request mapping + "model": model, + "file": audio_file, + "response_format": "json", + **optional_params, + } + return AudioTranscriptionRequestData(data=data) diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 48a44575989..eb523f3f050 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -25,6 +25,23 @@ from ..openai import OpenAIChatCompletion class OpenAIAudioTranscription(OpenAIChatCompletion): # Audio Transcriptions + @staticmethod + def _sdk_compatible_request_data(data: dict) -> dict: + """Route API fields that predate SDK support through ``extra_body``.""" + extension_keys: Final = ("keywords", "languages") + extension_body: Final = {key: data[key] for key in extension_keys if key in data} + if not extension_body: + return data + + existing_extra_body: Final = data.get("extra_body") + return { # mutable-ok: OpenAI SDK requires a mutable request mapping + **{key: value for key, value in data.items() if key not in extension_keys}, + "extra_body": { + **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), + **extension_body, + }, + } + async def make_openai_audio_transcriptions_request( self, openai_aclient: AsyncOpenAI, @@ -32,11 +49,17 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout: float | httpx.Timeout, ): try: - raw_response = await openai_aclient.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) + sdk_data: Final = self._sdk_compatible_request_data(data) + if data.get("stream") is True: + stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout) + return {}, stream_response # mutable-ok: response headers use the existing mutable mapping contract + raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( + **sdk_data, timeout=timeout + ) # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() + parsed_response: Final = raw_response.parse() - return headers, response + return headers, parsed_response except Exception as e: raise e @@ -47,7 +70,11 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout: float | httpx.Timeout, ): try: - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) + sdk_data: Final = self._sdk_compatible_request_data(data) + if data.get("stream") is True: + response = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) + return None, response + raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**sdk_data, timeout=timeout) headers: Final = dict(raw_response.headers) response: Final = raw_response.parse() return headers, response @@ -128,6 +155,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): ) logging_obj.model_call_details["response_headers"] = headers + if data.get("stream") is True: + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -192,6 +222,8 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, ) logging_obj.model_call_details["response_headers"] = headers + if data.get("stream") is True: + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() else: diff --git a/litellm/main.py b/litellm/main.py index 27700cf9def..94aaa1383b0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -36,6 +36,8 @@ if TYPE_CHECKING: import dotenv import httpx import openai +from openai import AsyncStream, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import assert_never, overload @@ -8165,7 +8167,10 @@ async def amoderation( @client -async def atranscription(*args, **kwargs) -> TranscriptionResponse: +async def atranscription( + *args, # noqa: ANN002 # public SDK wrapper preserves positional call compatibility + **kwargs, # noqa: ANN003 # kwargs-ok: public SDK wrapper preserves keyword call compatibility +) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: """ Calls openai + azure whisper endpoints. @@ -8209,6 +8214,12 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: else: # Call the synchronous function using run_in_executor response = await loop.run_in_executor(None, func_with_context) + if kwargs.get("stream") is True and isinstance(response, AsyncStream): + if file is not None: + calculated_duration = calculate_request_duration(file) + if calculated_duration is not None: + setattr(response, "_litellm_audio_duration", calculated_duration) + return response if not isinstance(response, TranscriptionResponse): raise ValueError( f"Invalid response from transcription provider, expected TranscriptionResponse, but got {type(response)}" @@ -8221,9 +8232,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: if response is not None and not isinstance(response, Coroutine) and file is not None: existing_duration: Final = getattr(response, "duration", None) if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) - if calculated_duration is not None: - response.hidden_params["audio_transcription_duration"] = calculated_duration + sync_calculated_duration: Final = calculate_request_duration(file) + if sync_calculated_duration is not None: + response.set_audio_transcription_duration(sync_calculated_duration) return response except Exception as e: @@ -8237,16 +8248,52 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: ) +def _validate_gpt_transcription_request( + model: str, + custom_llm_provider: str, + language: str | None, + languages: Sequence[str] | None, + response_format: str | None, + api_version: str | None, +) -> str | None: + if language is not None and languages is not None: + raise litellm.UnsupportedParamsError( + message="language and languages cannot be used together", + model=model, + llm_provider=custom_llm_provider, + ) + if model == "gpt-live-transcribe": + raise litellm.UnsupportedParamsError( + message="gpt-live-transcribe is available through the Realtime API, not file transcription", + model=model, + llm_provider=custom_llm_provider, + ) + if model == "gpt-transcribe" and response_format not in (None, "json"): + raise litellm.UnsupportedParamsError( + message="gpt-transcribe only supports response_format='json'", + model=model, + llm_provider=custom_llm_provider, + ) + if custom_llm_provider == "azure" and model == "gpt-transcribe": + if api_version in (None, "v1", "latest", "preview"): + return litellm.AZURE_DEFAULT_API_VERSION + return api_version + return api_version + + @client def transcription( model: str, file: FileTypes, ## OPTIONAL OPENAI PARAMS ## language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: Literal["json", "text", "srt", "verbose_json", "vtt"] | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, temperature: int | None = None, # openai defaults this to 0 + stream: bool | None = None, ## LITELLM PARAMS ## user: str | None = None, timeout=600, # default to 10 minutes @@ -8257,7 +8304,11 @@ def transcription( custom_llm_provider=None, base_url: str | None = None, **kwargs, -) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: +) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] +): """ Calls openai + azure whisper endpoints. @@ -8295,13 +8346,25 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key + validated_api_version: Final = _validate_gpt_transcription_request( + model=model, + custom_llm_provider=custom_llm_provider, + language=language, + languages=languages, + response_format=response_format, + api_version=api_version, + ) + optional_params: Final = get_optional_params_transcription( model=model, language=language, + languages=languages, + keywords=keywords, prompt=prompt, response_format=response_format, timestamp_granularities=timestamp_granularities, temperature=temperature, + stream=stream, custom_llm_provider=custom_llm_provider, **non_default_params, ) @@ -8324,7 +8387,13 @@ def transcription( custom_llm_provider=custom_llm_provider, ) - response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None + response: ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | AsyncStream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + | None + ) = None provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model, @@ -8339,7 +8408,7 @@ def transcription( # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + azure_api_version: Final = validated_api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN") @@ -8358,7 +8427,7 @@ def transcription( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - api_version=api_version, + api_version=azure_api_version, azure_ad_token=azure_ad_token, max_retries=max_retries, litellm_params=litellm_params_dict, @@ -8490,14 +8559,12 @@ def transcription( # Store duration in _hidden_params for cost calculation without # exposing it in the response body (see sync path comment above). if response is not None and not isinstance(response, Coroutine): - existing_duration: Final = getattr(response, "duration", None) - if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) + calculated_duration: Final = calculate_request_duration(file) + if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) - ) - response_hidden_params["audio_transcription_duration"] = calculated_duration + setattr(response, "_litellm_audio_duration", calculated_duration) + elif getattr(response, "duration", None) is None and calculated_duration is not None: + response.set_audio_transcription_duration(calculated_duration) if response is None: raise ValueError("Unmapped provider passed in. Unable to get the response.") diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 0e2eed73364..9942d6ea113 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -41,6 +41,20 @@ def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], return _register +def _include_router_prepend(attr_name: str = "router") -> Callable[["FastAPI", object], None]: + def _register(app: "FastAPI", module: object) -> None: + router: Final = app.router + route_count: Final = len(router.routes) + app.include_router(getattr(module, attr_name)) + new_routes: Final = router.routes[route_count:] + router.routes[:] = [ # mutable-ok: FastAPI route registration requires in-place list replacement + *new_routes, + *router.routes[:route_count], + ] + + return _register + + def _mount_app(prefix: str, attr_name: str = "app") -> Callable[["FastAPI", object], None]: def _register(app: "FastAPI", module: object) -> None: app.mount(path=prefix, app=getattr(module, attr_name)) @@ -246,6 +260,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( name="realtime", module_path="litellm.proxy.realtime_endpoints.endpoints", path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"), + register_fn=_include_router_prepend(), ), LazyFeature( name="anthropic_passthrough", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4572e59944b..b3d9c04f4da 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -33217,6 +33217,51 @@ ] } }, + "/openai/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "llm_passthrough" + ] + } + }, + "/openai/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "llm_passthrough" + ] + } + }, "/openai/v1/responses": { "post": { "description": "Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses\n\nSupports background mode with polling_via_cache for partial response retrieval.\nWhen background=true and polling_via_cache is enabled, returns a polling_id immediately\nand streams the response in the background, updating Redis cache.\n\n```bash\n# Normal request\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\"\n}'\n\n# Background request with polling\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\",\n \"background\": true\n}'\n```", @@ -53360,6 +53405,51 @@ ] } }, + "/openai/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post_2", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/openai/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post_2", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/realtime/calls": { "post": { "operationId": "proxy_realtime_calls_realtime_calls_post", @@ -53432,6 +53522,51 @@ ] } }, + "/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/v1/realtime/calls": { "post": { "operationId": "proxy_realtime_calls_v1_realtime_calls_post", @@ -53503,6 +53638,51 @@ "realtime" ] } + }, + "/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_v1_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_v1_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 54acf59ef80..b22886a902a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -327,6 +327,9 @@ class LiteLLMRoutes(enum.Enum): "/realtime?{model}", "/v1/realtime?{model}", "/openai/v1/realtime?{model}", + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", # realtime (GA WebRTC HTTP routes) "/realtime/client_secrets", "/v1/realtime/client_secrets", @@ -337,6 +340,12 @@ class LiteLLMRoutes(enum.Enum): "/realtime/transcription_sessions", "/v1/realtime/transcription_sessions", "/openai/v1/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", # responses API "/responses", "/v1/responses", @@ -2484,6 +2493,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): admission_queue_timeout_seconds: float = Field( 1.0, gt=0, description="maximum time a request waits for a worker slot" ) + allow_non_billable_realtime_protocols: bool = Field( + False, + description="Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement", + ) plugins: list[PluginConfig] | None = Field( None, description="external services registered as embeddable UI plugins" ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 23faa964d5f..e0ae8ba1547 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -155,7 +155,9 @@ ProxyRouteType: TypeAlias = Literal[ "_arealtime", "_aresponses_websocket", "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "aget_responses", "adelete_responses", "acancel_responses", diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 245a56678eb..56a46de67f2 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -523,14 +523,18 @@ async def get_form_data(request: Request) -> dict[str, Any]: Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` """ form: Final = await request.form() - parsed_form_data: Final[dict[str, Any]] = {} - for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat - if key.endswith("[]"): - clean_key = key[:-2] - parsed_form_data.setdefault(clean_key, []).append(value) - else: - parsed_form_data[key] = value - return parsed_form_data + form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items()) + array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]")) + normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items) + normalized_keys: Final = frozenset(key for key, _ in normalized_items) + return { # mutable-ok: request parsers expose a mutable form-data mapping to endpoint handlers + key: [ # mutable-ok: repeated multipart values follow the established mutable list contract + value for item_key, value in normalized_items if item_key == key + ] + if key in array_keys + else next(value for item_key, value in reversed(normalized_items) if item_key == key) + for key in normalized_keys + } async def convert_upload_files_to_file_data( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f632317d9a8..668cb15ac65 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -58,6 +58,7 @@ import anyio import anyio.lowlevel import websockets import websockets.exceptions +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config from pydantic.fields import FieldInfo, PydanticUndefined from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never @@ -13379,7 +13380,10 @@ async def audio_transcriptions( try: # Use orjson to parse JSON data, orjson speeds up requests significantly form_data: Final = await get_form_data(request) - data = {**_form_fields_without_file(form_data), **data} + data = { + key: value is True or str(value).lower() in ("1", "true") if key == "stream" else value + for key, value in _form_fields_without_file(form_data).items() + } | data # Include original request and headers in the data data = await add_litellm_data_to_request( @@ -13439,6 +13443,29 @@ async def audio_transcriptions( finally: file_object.close() # close the file read in by io library + if data.get("stream") is True: + if not hasattr(response, "__aiter__"): + raise TypeError(f"Streaming transcription returned {type(response).__name__}, expected an async stream") + stream_response: Final = cast(AsyncIterator[TranscriptionStreamEvent], response) + + async def transcription_event_stream( + stream: AsyncIterator[TranscriptionStreamEvent], + ) -> AsyncGenerator[str, None]: + try: + async for event in stream: + yield f"data: {event.model_dump_json()}\n\n" + finally: + close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None) + if callable(close): + close_result: Final = close() + if inspect.isawaitable(close_result): + await close_result + + return StreamingResponse( + transcription_event_stream(stream_response), + media_type="text/event-stream", + ) + ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) @@ -13622,9 +13649,39 @@ async def _reject_realtime_session( await _release_realtime_max_parallel_slot(user_api_key_dict) +def _resolve_realtime_route_model( + model: str | None, + intent: str | None, + is_translation: bool, +) -> str | None: + if model is not None: + return model + if is_translation: + return "gpt-realtime-translate" + if intent == "transcription": + return "gpt-realtime-whisper" + return None + + +def _resolve_realtime_upstream_query_model( + model: str | None, + intent: str | None, + is_translation: bool, + route_model: str, +) -> str | None: + if intent == "transcription": + return None + if is_translation: + return route_model + return model + + @app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") +@app.websocket("/openai/v1/realtime/translations") +@app.websocket("/v1/realtime/translations") +@app.websocket("/realtime/translations") async def realtime_websocket_endpoint( websocket: WebSocket, model: str | None = fastapi.Query(None, description="The model to use for the websocket connection."), @@ -13641,15 +13698,13 @@ async def realtime_websocket_endpoint( if protocol.strip() ) - route_model = model + is_translation: Final = websocket.url.path.endswith("/realtime/translations") + route_model: Final = _resolve_realtime_route_model(model, intent, is_translation) if route_model is None: - if intent == "transcription": - route_model = "gpt-realtime-whisper" - else: - await _reject_realtime_session( - websocket, user_api_key_dict, code=1008, reason="model query parameter is required" - ) - return + await _reject_realtime_session( + websocket, user_api_key_dict, code=1008, reason="model query parameter is required" + ) + return assert route_model is not None try: await can_key_call_resolved_model( @@ -13685,11 +13740,24 @@ async def _route_realtime_websocket_session( live_session_start: Mapping[str, object] | None = None, ) -> None: # Only use explicit parameters, not all query params - query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent))) + query_model: Final = _resolve_realtime_upstream_query_model( + model=model, + intent=intent, + is_translation=is_translation, + route_model=route_model, + ) + query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys + RealtimeQueryParams, + dict( # mutable-ok: downstream realtime routing normalizes this request-scoped query mapping + _realtime_query_params_template(query_model, intent) + ), + ) + data: dict[str, object] = { "model": route_model, "websocket": websocket, "query_params": query_params, # Only explicit params + "realtime_mode": "translation" if is_translation else "realtime", } # Pass guardrails into data so pre-call guardrail processing picks them up if guardrails: diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 9c0f3c4b81c..e3038c21a6d 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -41,7 +41,21 @@ router: Final = APIRouter() _REALTIME_TOKEN_VERSION: Final = "realtime_v1" _DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview" _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" -_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription") +_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation") +_NON_BILLABLE_REALTIME_PROTOCOL_SETTING: Final = "allow_non_billable_realtime_protocols" +_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE: Final = ( + "Realtime WebRTC endpoints are disabled because provider usage bypasses LiteLLM billing. " + "Set general_settings.allow_non_billable_realtime_protocols to true to opt in" +) + + +def _enforce_non_billable_realtime_protocol_gate(general_settings: Mapping[str, object]) -> None: + if general_settings.get(_NON_BILLABLE_REALTIME_PROTOCOL_SETTING) is True: + return + raise HTTPException( + status_code=http_status.HTTP_403_FORBIDDEN, + detail=_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE, + ) def _coerce_realtime_session_type(session_type: str | None) -> str: @@ -124,19 +138,47 @@ def _set_transcription_model_on_session( } +async def _authorize_and_bind_nested_transcription_models( + session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization + user_api_key_dict: UserAPIKeyAuth, + llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list + llm_router: Any, +) -> None: + nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data)) + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + if nested_models: + _set_transcription_model_on_session( + session=session_data, + model=nested_models[0], + ) + + async def _prepare_client_secret_session( req: RealtimeClientSecretRequest, user_api_key_dict: UserAPIKeyAuth, llm_model_list: list | None, llm_router: "Router | None", + forced_session_type: str | None = None, ) -> tuple[str, dict | None, str]: - session_type: Final = _coerce_realtime_session_type(req.session.type if req.session else None) - session_data: Final[dict | None] = req.session.model_dump(exclude_none=True) if req.session else None + requested_session_type: Final = req.session.type if req.session else None + if forced_session_type is None and requested_session_type == "translation": + raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint") + session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type) + session_data: Final[dict | None] = ( + req.session.model_dump(exclude_none=True) if req.session else ({} if session_type == "translation" else None) + ) if session_data is not None: session_data["type"] = session_type session_model: Final = req.session.model if req.session else None - model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL + default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL + model: str = session_model or req.model or default_model if session_type != "transcription": await can_key_call_resolved_model( model=model, @@ -144,6 +186,15 @@ async def _prepare_client_secret_session( llm_model_list=llm_model_list, llm_router=llm_router, ) + if session_data is not None: + session_data["model"] = model + if session_type == "translation": + await _authorize_and_bind_nested_transcription_models( + session_data=session_data, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) return model, session_data, session_type transcription_model_candidates: Final = _transcription_model_candidates_from_session(session_data or {}) @@ -236,6 +287,21 @@ def _decode_realtime_token_payload( dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/openai/v1/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) async def create_realtime_client_secret( request: Request, fastapi_response: Response, @@ -253,16 +319,19 @@ async def create_realtime_client_secret( version, ) + _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await read_request_body(request=request) req: Final = RealtimeClientSecretRequest(**body) + is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path model, session_data, session_type = await _prepare_client_secret_session( req=req, user_api_key_dict=user_api_key_dict, llm_model_list=llm_model_list, llm_router=llm_router, + forced_session_type="translation" if is_translation_request else None, ) data = {"model": model} @@ -286,17 +355,20 @@ async def create_realtime_client_secret( proxy_config=proxy_config, ) + call_type: Final = ( + "acreate_realtime_translation_client_secret" if is_translation_request else "acreate_realtime_client_secret" + ) data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=data, - call_type="acreate_realtime_client_secret", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/client_secrets (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="acreate_realtime_client_secret", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -379,6 +451,18 @@ async def create_realtime_client_secret( "/openai/v1/realtime/calls", tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/openai/v1/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) async def proxy_realtime_calls( request: Request, fastapi_response: Response, @@ -404,6 +488,7 @@ async def proxy_realtime_calls( media_type="application/json", ) + is_translation_request: Final = "/realtime/translations/calls" in request.url.path encrypted_token: Final = auth_header.removeprefix("Bearer ").strip() decrypted_token_value: Final = decrypt_value_helper( value=encrypted_token, @@ -416,26 +501,42 @@ async def proxy_realtime_calls( media_type="application/json", ) + _enforce_non_billable_realtime_protocol_gate(general_settings) sdp_body: Final[bytes] = await request.body() decoded_payload: Final = _decode_realtime_token_payload(decrypted_token_value) if decoded_payload is not None: # Check token expiry expires_at: Final = decoded_payload.get("expires_at") - if expires_at is not None and isinstance(expires_at, int): - if time.time() > expires_at: - return Response( - content=json.dumps({"error": "Token has expired"}), - status_code=http_status.HTTP_401_UNAUTHORIZED, - media_type="application/json", - ) + if isinstance(expires_at, int) and time.time() > expires_at: + return Response( + content=json.dumps({"error": "Token has expired"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decoded_payload.get("ephemeral_key", "") model = decoded_payload.get("model_id") or request.query_params.get("model") or _DEFAULT_REALTIME_MODEL user_id = decoded_payload.get("user_id") or None team_id = decoded_payload.get("team_id") or None - session_type = _coerce_realtime_session_type(decoded_payload.get("session_type")) + raw_session_type: Final = decoded_payload.get("session_type") + session_type = _coerce_realtime_session_type(raw_session_type) + if is_translation_request != (raw_session_type == "translation"): + return Response( + content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping + {"error": "Token is not valid for this Realtime endpoint"} + ), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) else: - # Backward compatibility: older tokens contained only encrypted upstream key. + if is_translation_request: + return Response( + content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping + {"error": "Token is not valid for this Realtime endpoint"} + ), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decrypted_token_value model = request.query_params.get("model", _DEFAULT_REALTIME_MODEL) user_id = None @@ -479,17 +580,18 @@ async def proxy_realtime_calls( proxy_config=proxy_config, ) + call_type: Final = "arealtime_translation_calls" if is_translation_request else "arealtime_calls" data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=minimal_auth, data=data, - call_type="arealtime_calls", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/calls (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="arealtime_calls", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -565,6 +667,7 @@ async def create_realtime_transcription_session( version, ) + _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await read_request_body(request=request) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index cb226929167..7c4b494813e 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -589,7 +589,9 @@ RouteType = Literal[ "alist_input_items", "_arealtime", # private function for realtime API "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "acreate_realtime_transcription_session", "_aresponses_websocket", # private function for responses WebSocket mode "aimage_edit", diff --git a/litellm/realtime_api/README.md b/litellm/realtime_api/README.md index d810de2f24f..d9699f57da7 100644 --- a/litellm/realtime_api/README.md +++ b/litellm/realtime_api/README.md @@ -6,4 +6,16 @@ Supported endpoints: Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI. -For user-facing documentation and usage examples, see the litellm-docs repo. \ No newline at end of file +Billing visibility: +- WebSocket sessions pass provider usage events through LiteLLM and support local spend tracking +- Client-secret and SDP call endpoints only proxy session setup; subsequent WebRTC media and usage events travel over the peer connection, so LiteLLM cannot record inference spend or enforce spend-based budgets for those sessions +- Use the proxied WebSocket transport when LiteLLM spend logs and budgets must include Realtime inference + +Non-billable Realtime protocols are disabled by default. Operators who accept the billing and budget-enforcement limitation can opt in: + +```yaml +general_settings: + allow_non_billable_realtime_protocols: true +``` + +For user-facing documentation and usage examples, see the litellm-docs repo. diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 088dfb54155..d658cf787d2 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -9,14 +9,18 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast from typing_extensions import ReadOnly, TypedDict, assert_never +import httpx + import litellm from litellm.constants import ( + AZURE_GA_REALTIME_MODELS, AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.xai.common_utils import XAIModelInfo @@ -78,6 +82,30 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: + if session.get("type") == "transcription": + audio = session.get("audio") + audio = audio if isinstance(audio, dict) else {} # mutable-ok: nested session model is rebuilt locally + audio_input = audio.get("input") + audio_input = ( # mutable-ok: nested session model is rebuilt locally + audio_input if isinstance(audio_input, dict) else {} + ) + transcription = audio_input.get("transcription") + transcription = ( # mutable-ok: nested session model is rebuilt locally + transcription if isinstance(transcription, dict) else {} + ) + return { # mutable-ok: provider routing requires an independently mutable session payload + **session, + "audio": { # mutable-ok: provider routing rebuilds nested audio configuration + **audio, + "input": { # mutable-ok: provider routing rebuilds nested input configuration + **audio_input, + "transcription": { # mutable-ok: resolved deployment replaces only the transcription model + **transcription, + "model": model_name, + }, + }, + }, + } if "model" not in session: return session return {**session, "model": model_name} @@ -92,6 +120,21 @@ def _build_litellm_metadata(kwargs: dict) -> dict[str, object]: return metadata +def _resolve_azure_realtime_protocol( + model: str, + realtime_protocol: str | None, + query_params: RealtimeQueryParams | None, + realtime_mode: str, +) -> str: + if model in AZURE_GA_REALTIME_MODELS: + if realtime_protocol is not None and realtime_protocol.upper() not in ("GA", "V1"): + raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") + return "GA" + if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": + return "GA" + return realtime_protocol or "beta" + + def _get_realtime_http_provider_config( custom_llm_provider: str, dynamic_api_base: str | None, @@ -105,10 +148,6 @@ def _get_realtime_http_provider_config( Uses ProviderConfigManager so each provider keeps its credential-resolution and URL-construction logic in its own transformation class. """ - from litellm.llms.base_llm.realtime.http_transformation import ( - BaseRealtimeHTTPConfig, - ) - provider_config: BaseRealtimeHTTPConfig | None = None if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_realtime_http_config( @@ -143,6 +182,22 @@ async def _realtime_http_bearer_token( return bearer_token or "" +def _get_realtime_http_extra_headers( + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + resolved_api_key: str, + extra_headers: Mapping[str, object] | None, +) -> Mapping[str, object] | None: + resolved_headers: Final = { # mutable-ok: Azure authentication may extend caller-supplied headers + **(extra_headers or {}) + } + if custom_llm_provider == "azure" and not resolved_api_key: + azure_ad_token: Final = get_azure_ad_token(litellm_params) + if azure_ad_token: + resolved_headers["Authorization"] = f"Bearer {azure_ad_token}" + return resolved_headers or None + + @wrapper_client async def acreate_realtime_client_secret( model: str | None = None, @@ -156,31 +211,49 @@ async def acreate_realtime_client_secret( session=RealtimeSessionConfig.model_validate(session) if session else None, expires_after=RealtimeExpiresAfter.model_validate(expires_after) if expires_after else None, ) - model_name = (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview" + transcription_model: Final = ( + req.session.audio.input.transcription.model + if req.session is not None + and req.session.audio is not None + and req.session.audio.input is not None + and req.session.audio.input.transcription is not None + else None + ) + provider_qualified_model: Final = ( + req.model + if req.model is not None + and "/" in req.model + and req.model.split("/", 1)[0] in LlmProviders._member_map_.values() + else None + ) + requested_model_name: Final = ( + provider_qualified_model + or transcription_model + or (req.session.model if req.session is not None else None) + or req.model + or "gpt-4o-realtime-preview" + ) litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj") litellm_params: Final = GenericLiteLLMParams(**kwargs) - ( - model_name, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = get_llm_provider( - model=model_name, + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, custom_llm_provider=litellm_params.custom_llm_provider, api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) - ( - provider_config, - resolved_api_base, - resolved_api_key, - ) = _get_realtime_http_provider_config( + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( custom_llm_provider=custom_llm_provider, dynamic_api_base=dynamic_api_base, dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -191,6 +264,11 @@ async def acreate_realtime_client_secret( request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) if isinstance(request_data.get("session"), dict): request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + elif req.model is not None: + request_data["session"] = { # mutable-ok: OpenAI SDK consumes this request-scoped session payload + "type": "realtime", + "model": model_name, + } return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( @@ -201,9 +279,86 @@ async def acreate_realtime_client_secret( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", + ) + + +@wrapper_client +async def acreate_realtime_translation_client_secret( + model: str | None = None, + session: Mapping[str, Any] | None = None, + expires_after: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options +) -> httpx.Response: + requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate" + session_config: Final = RealtimeSessionConfig.model_validate( + { # mutable-ok: Pydantic validates this request-scoped translation session payload + **(session or {}), + "type": "translation", + "model": requested_model_name, + } + ) + req: Final = RealtimeClientSecretRequest( + model=requested_model_name, + session=session_config, + expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None, + ) + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ # mutable-ok: logging owns a mutable request metadata payload + "expires_after": expires_after, + "session": session, + }, + litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload + "api_base": resolved_api_base + }, + custom_llm_provider=custom_llm_provider, + ) + request_data: Final = req.model_dump( + exclude_none=True, + exclude={"model"}, # mutable-ok: Pydantic requires a mutable field-exclusion set + ) + request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + request_data["session"].pop("type", None) + return await base_llm_http_handler.async_realtime_translation_client_secret_handler( + api_base=resolved_api_base, + api_key=resolved_api_key, + request_data=request_data, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + extra_headers=resolved_extra_headers, + client=kwargs.get("client"), + api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", ) @@ -252,6 +407,12 @@ async def acreate_realtime_transcription_session( dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -276,7 +437,7 @@ async def acreate_realtime_transcription_session( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, ) @@ -333,6 +494,70 @@ async def arealtime_calls( extra_headers=kwargs.get("extra_headers"), client=kwargs.get("client"), api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", + ) + + +@wrapper_client +async def arealtime_translation_calls( + openai_ephemeral_key: str, + sdp_body: bytes, + model: str | None = None, + session: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options +) -> httpx.Response: + requested_model_name: Final = model or "gpt-realtime-translate" + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + session_config: Final = _with_resolved_session_model( + { # mutable-ok: provider routing requires an independently mutable session payload + **(session or {}), + "type": "translation", + "model": model_name, + }, + model_name, + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ # mutable-ok: logging owns a mutable request metadata payload + "realtime_translation_calls": True, + "session": session_config, + }, + litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload + "api_base": resolved_api_base + }, + custom_llm_provider=custom_llm_provider, + ) + return await base_llm_http_handler.async_realtime_calls_handler( + api_base=resolved_api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_body=sdp_body, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + session_config=session_config, + extra_headers=kwargs.get("extra_headers"), + client=kwargs.get("client"), + api_version=litellm_params.api_version, + translation=True, + use_openai_sdk=custom_llm_provider == "openai", ) @@ -465,6 +690,8 @@ async def _arealtime( timeout: float | None = None, query_params: RealtimeQueryParams | None = None, live_session_start: Mapping[str, object] | None = None, + + realtime_mode: str = "realtime", **kwargs, ) -> None: """ @@ -558,9 +785,6 @@ async def _arealtime( or litellm_params.get("realtime_protocol") or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") ) - realtime_protocol: Final = azure_realtime_protocol_for_client( - configured_realtime_protocol, query_params=query_params, websocket=websocket - ) resolved_azure_ad_token: Final = ( None if api_key @@ -569,6 +793,18 @@ async def _arealtime( GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), ) ) + realtime_protocol: Final = _resolve_azure_realtime_protocol( + model=model, + realtime_protocol=( + configured_realtime_protocol + if model in AZURE_GA_REALTIME_MODELS or realtime_mode == "translation" + else azure_realtime_protocol_for_client( + configured_realtime_protocol, query_params=query_params, websocket=websocket + ) + ), + query_params=query_params, + realtime_mode=realtime_mode, + ) await azure_realtime.async_realtime( model=model, websocket=websocket, @@ -581,6 +817,7 @@ async def _arealtime( logging_obj=litellm_logging_obj, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), litellm_metadata=_build_litellm_metadata(kwargs), ) @@ -596,9 +833,10 @@ async def _arealtime( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - client=None, + client=client, timeout=timeout, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), litellm_metadata=_build_litellm_metadata(kwargs), ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0d59c93786c..8b4863d7b22 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -227,6 +227,8 @@ _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( "insufficient_quota": 429, "vector_store_timeout": 504, "invalid_prompt": 400, + "data_residency_mismatch": 400, + "bio_policy": 400, "invalid_image": 400, "invalid_image_format": 400, "invalid_base64_image": 400, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 503d85aaed9..a6e87d0a567 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1251,12 +1251,14 @@ class ResponseAPILoggingUtils: else: prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), + cached_tokens_details=getattr( + response_api_usage.input_tokens_details, + "cached_tokens_details", + None, + ), audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), - cached_tokens_details=getattr( - response_api_usage.input_tokens_details, "cached_tokens_details", None - ), video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), cache_type=getattr(response_api_usage.input_tokens_details, "cache_type", None), diff --git a/litellm/router.py b/litellm/router.py index da04e59f18a..39c0bcda338 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2465,6 +2465,14 @@ class Router: self.acreate_realtime_transcription_session = self.factory_function( litellm.acreate_realtime_transcription_session, call_type="acreate_realtime_transcription_session" ) + self.acreate_realtime_translation_client_secret = self.factory_function( + litellm.acreate_realtime_translation_client_secret, + call_type="acreate_realtime_translation_client_secret", + ) + self.arealtime_translation_calls = self.factory_function( + litellm.arealtime_translation_calls, + call_type="arealtime_translation_calls", + ) self._aresponses_websocket = self.factory_function( litellm._aresponses_websocket, call_type="_aresponses_websocket" ) @@ -7783,6 +7791,8 @@ class Router: "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", "_aresponses_websocket", "acreate_fine_tuning_job", "acancel_fine_tuning_job", @@ -8053,6 +8063,8 @@ class Router: "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 62dc152ca84..4c0e6a78d4b 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1216,9 +1216,12 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]] OpenAIAudioTranscriptionOptionalParams = Literal[ "language", + "languages", + "keywords", "prompt", "temperature", "response_format", + "stream", "timestamp_granularities", "include", ] @@ -2438,6 +2441,16 @@ class OpenAIRealtimeResponseUsage(TypedDict): output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] +class OpenAIRealtimeTranslationDurationUsage(TypedDict): + type: ReadOnly[Literal["duration"]] + output_seconds: ReadOnly[float] + + +class OpenAIRealtimeTranslationClosedEvent(TypedDict): + type: ReadOnly[Literal["session.closed"]] + usage: ReadOnly[OpenAIRealtimeTranslationDurationUsage] + + class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" # Beta delta event names @@ -2481,6 +2494,7 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeInputAudioTranscriptionCompleted | OpenAIRealtimeTranscriptionSessionCreated | OpenAIRealtimeErrorEvent + | OpenAIRealtimeTranslationClosedEvent ) OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents] diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 6da73982291..fafa93152e6 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,6 +1,7 @@ -from typing import Any, Literal +from collections.abc import Mapping, Sequence +from typing import Any, Final, Literal -from pydantic import ConfigDict +from pydantic import BaseModel, ConfigDict from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -12,6 +13,7 @@ from .llms.openai import ( ) ALL_DELTA_TYPES = Literal["text", "audio"] +RealtimeSessionType: Final = Literal["realtime", "transcription", "translation"] class RealtimeResponseTransformInput(TypedDict): @@ -66,7 +68,42 @@ class RealtimeExpiresAfter(LiteLLMBaseModel): seconds: int | None = None -class RealtimeSessionConfig(LiteLLMBaseModel): +class RealtimeAudioTranscriptionConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + model: str | None = None + delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None + keywords: Sequence[str] | None = None + language: str | None = None + languages: Sequence[str] | None = None + prompt: str | None = None + + +class RealtimeAudioInputConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + format: str | Mapping[str, object] | None = None + noise_reduction: Mapping[str, object] | None = None + transcription: RealtimeAudioTranscriptionConfig | None = None + turn_detection: Mapping[str, object] | None = None + + +class RealtimeAudioOutputConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + format: str | Mapping[str, object] | None = None + language: str | None = None + voice: str | Mapping[str, object] | None = None + + +class RealtimeSessionAudioConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + input: RealtimeAudioInputConfig | None = None + output: RealtimeAudioOutputConfig | None = None + + +class RealtimeSessionConfig(BaseModel): """ Session configuration nested inside the client_secrets request body. @@ -77,10 +114,10 @@ class RealtimeSessionConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="allow") - type: str | None = None + type: RealtimeSessionType | None = None model: str | None = None instructions: str | None = None - audio: dict[str, object] | None = None + audio: RealtimeSessionAudioConfig | None = None include: list[str] | None = None max_output_tokens: int | str | None = None output_modalities: list[str] | None = None @@ -134,12 +171,15 @@ class RealtimeTranscriptionSessionRequest(LiteLLMBaseModel): # LiteLLM-only routing hint — stripped before forwarding upstream. model: str | None = None input_audio_transcription: dict[str, Any] | None = None + audio: RealtimeSessionAudioConfig | None = None def resolved_model(self) -> str | None: if self.model: return self.model if self.input_audio_transcription: return self.input_audio_transcription.get("model") + if self.audio and self.audio.input and self.audio.input.transcription: + return self.audio.input.transcription.model return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..a9ef1f04a8e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -524,6 +524,11 @@ class CallTypes(str, Enum): aextraction = "aextraction" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" + acreate_realtime_client_secret = "acreate_realtime_client_secret" + arealtime_calls = "arealtime_calls" + acreate_realtime_transcription_session = "acreate_realtime_transcription_session" + acreate_realtime_translation_client_secret = "acreate_realtime_translation_client_secret" + arealtime_translation_calls = "arealtime_translation_calls" create_batch = "create_batch" acreate_batch = "acreate_batch" aretrieve_batch = "aretrieve_batch" @@ -758,6 +763,8 @@ CallTypesLiteral = Literal[ "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ] MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset( @@ -766,6 +773,12 @@ MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset( # Mapping of API routes to their corresponding call types API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { + "/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/openai/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/openai/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), # Chat Completions "/chat/completions": [CallTypes.acompletion, CallTypes.completion], "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], @@ -1084,9 +1097,12 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API - "/realtime": [CallTypes.arealtime], - "/v1/realtime": [CallTypes.arealtime], - "/openai/v1/realtime": [CallTypes.arealtime], + "/realtime": (CallTypes.arealtime,), + "/v1/realtime": (CallTypes.arealtime,), + "/openai/v1/realtime": (CallTypes.arealtime,), + "/realtime/translations": (CallTypes.arealtime,), + "/v1/realtime/translations": (CallTypes.arealtime,), + "/openai/v1/realtime/translations": (CallTypes.arealtime,), "/live/sessions": (CallTypes.arealtime,), "/v1/live/sessions": (CallTypes.arealtime,), "/openai/v1/live/sessions": (CallTypes.arealtime,), @@ -1816,6 +1832,8 @@ class PromptTokensDetailsWrapper( image_tokens: int | None = None """Image tokens sent to the model.""" + cached_tokens_details: CachedTokensDetails | None = None + video_tokens: int | None = None """Video tokens sent to the model.""" @@ -2859,12 +2877,17 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_tokens: int output_tokens: int total_tokens: int - input_token_details: TranscriptionUsageInputTokenDetailsObject + input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None + + +class TranscriptionDetectedLanguage(BaseModel): + code: str class TranscriptionResponse(OpenAIObject): text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None + languages: Sequence[TranscriptionDetectedLanguage] | None = None _hidden_params: dict = {} _response_headers: dict | None = None @@ -2877,8 +2900,11 @@ class TranscriptionResponse(OpenAIObject): def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params - def __init__(self, text=None) -> None: - super().__init__(text=text) + def __init__(self, text=None, usage=None, languages=None, **kwargs) -> None: # noqa: ANN003 # OpenAI-compatible response accepts provider extension fields + super().__init__(text=text, usage=usage, languages=languages, **kwargs) + + def set_audio_transcription_duration(self, duration: float) -> None: + self._hidden_params["audio_transcription_duration"] = duration def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/litellm/utils.py b/litellm/utils.py index 7de7ce6188d..66505a389f5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1431,6 +1431,8 @@ def function_setup( applied_guardrails=applied_guardrails, supports_correlation_logging=is_async_call, ) + if logging_obj is None: + raise RuntimeError("LiteLLM logging initialization returned no logger") ## check if metadata is passed in litellm_params: Final[dict[str, object]] = {"api_base": ""} @@ -2017,6 +2019,12 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.transcription.value and isinstance(result, openai.Stream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + wrap_transcription_stream, + ) + + result = wrap_transcription_stream(result, logging_obj, start_time) # RETURN RESULT update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata update_response_metadata( @@ -2318,6 +2326,12 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.atranscription.value and isinstance(result, openai.AsyncStream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + wrap_transcription_stream, + ) + + result = wrap_transcription_stream(result, logging_obj, start_time) _update_response_metadata( result=result, logging_obj=logging_obj, @@ -4026,10 +4040,13 @@ def get_optional_params_transcription( model: str, custom_llm_provider: str, language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: str | None = None, temperature: int | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, + stream: bool | None = None, drop_params: bool | None = None, **kwargs, ): @@ -4039,19 +4056,22 @@ def get_optional_params_transcription( passed_params: Final = locals() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") - passed_params.pop("custom_llm_provider") - passed_params.pop("drop_params") - drop_params = normalize_drop_params(drop_params) + passed_params.pop("model") + custom_llm_provider = passed_params.pop("custom_llm_provider") + drop_params = normalize_drop_params(passed_params.pop("drop_params")) special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs") for k, v in special_params.items(): passed_params[k] = v default_params: Final = { "language": None, + "languages": None, + "keywords": None, "prompt": None, "response_format": None, "temperature": None, # openai defaults this to 0 "timestamp_granularities": None, + "stream": None, } non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} @@ -4108,6 +4128,9 @@ def get_optional_params_transcription( openai_params=OPENAI_TRANSCRIPTION_PARAMS, additional_drop_params=kwargs.get("additional_drop_params", None), ) + extra_body: Final = optional_params.get("extra_body") + if isinstance(extra_body, dict) and not extra_body: + optional_params.pop("extra_body") return optional_params @@ -6763,6 +6786,7 @@ def get_model_info_helper( cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), + cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None @@ -9772,7 +9796,13 @@ class ProviderConfigManager: return XAIAudioTranscriptionConfig() elif litellm.LlmProviders.OPENAI == provider: - if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model): + if model == "gpt-transcribe": + from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, + ) + + return OpenAIGPTTranscribeAudioTranscriptionConfig() + elif "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model): return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 330ddf88fc5..bf3c5fe142f 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -76,6 +76,10 @@ - {id: llm.bedrock_native.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native invoke stream"} - {id: llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock invoke missing fields and invalid temperature"} - {id: llm.ocr.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: ocr, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.13 / LIT-4778", rationale: "OCR missing document rejected"} +- {id: llm.realtime.openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: translation, streaming: stream, assertions: [works], source: "realtime_endpoints/endpoints.py", rationale: "Dedicated translation client-secret, raw SDP, and WebSocket paths emit translated audio and transcript deltas"} +- {id: llm.realtime.azure_openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: translation, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA translation session emits translated audio and transcript deltas"} +- {id: llm.realtime.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/realtime/handler.py", rationale: "gpt-live-transcribe and gpt-realtime-whisper emit live transcript deltas"} +- {id: llm.realtime.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA live transcription emits transcript deltas"} - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} @@ -93,6 +97,8 @@ - {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"} - {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"} - {id: llm.audio_transcriptions.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.7 / LIT-4778", rationale: "Transcription empty file and missing model are rejected"} +- {id: llm.audio_transcriptions.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "gpt-transcribe streams typed transcript delta and done events"} +- {id: llm.audio_transcriptions.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure gpt-transcribe streams typed transcript delta and done events over the v1 API"} - {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"} - {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"} - {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index d2ba2c325df..47df7dae635 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -93,6 +93,9 @@ LlmCapability = Literal[ "tool_search_history", "tool_use", "upstream_stream_failure", + + "transcription", + "translation", "vision", "web_search", "web_search_server_tool", @@ -162,14 +165,7 @@ class OtherCell(_Base): Cell = Annotated[ - LlmCell - | MgmtCell - | McpCell - | ReliabilityCell - | QuotaCell - | LoggingCell - | GuardrailCell - | OtherCell, + LlmCell | MgmtCell | McpCell | ReliabilityCell | QuotaCell | LoggingCell | GuardrailCell | OtherCell, Field(discriminator="module"), ] diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 7d85da3f756..b62fceeb53e 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -27,6 +27,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter + +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeResponseTypedDict @@ -645,6 +647,42 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): assert "audio" not in session +@pytest.mark.asyncio +async def test_translation_session_update_omits_session_type(): + client_ws = MagicMock() + client_ws.scope = {"headers": []} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": {"output": {"language": "fr"}}, + }, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + translation_session=True, + ) + + await streaming.client_ack_messages() + + sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) + assert "type" not in sent_to_backend["session"] + assert sent_to_backend["session"]["audio"]["output"]["language"] == "fr" + + def test_translate_event_to_beta_renames_delta_types(): ev = RealTimeStreaming._translate_event_to_beta( {"type": "response.output_audio.delta", "delta": "abc", "event_id": "e1"} @@ -1157,6 +1195,93 @@ async def test_transcription_session_update_enforces_authorized_nested_model(): assert streaming._is_transcription_session is True +@pytest.mark.asyncio +async def test_translation_session_update_rejects_disallowed_nested_transcription_model() -> None: + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + translation_session=True, + ) + + with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe"}, + } + }, + }, + } + ) + ) + backend_ws.send.assert_not_awaited() + assert streaming._is_transcription_session is False + + +@pytest.mark.asyncio +async def test_translation_session_update_binds_nested_transcription_model() -> None: + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + translation_session=True, + ) + + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-realtime-whisper", "language": "en"}, + } + }, + }, + } + ) + ) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe", "language": "fr"}, + } + }, + }, + } + ) + ) + + first_sent = json.loads(backend_ws.send.await_args_list[0].args[0]) + second_sent = json.loads(backend_ws.send.await_args_list[1].args[0]) + assert first_sent["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + assert second_sent["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", + "language": "fr", + } + assert streaming._is_transcription_session is False + + @pytest.mark.asyncio async def test_normal_realtime_session_keeps_nested_transcription_model(): backend_ws = MagicMock() @@ -2918,6 +3043,69 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] +def test_translation_audio_duration_is_finalized_once(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + payload = base64.b64encode(bytes(48000)).decode() + streaming._capture_translation_output_audio({"type": "session.output_audio.delta", "delta": payload}) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +def test_translation_audio_duration_uses_session_output_format(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"output": {"format": {"type": "audio/pcmu", "rate": 8000}}}}, + } + ) + streaming._capture_translation_output_audio( + { + "type": "session.output_audio.delta", + "delta": base64.b64encode(bytes(8000)).decode(), + } + ) + streaming._finalize_translation_usage() + + assert streaming.messages[-1]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +def test_translation_does_not_duplicate_provider_duration_usage(): + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + streaming.messages.append({"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}}) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 1dc7ff74323..83f3be5562c 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -482,6 +482,11 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "avector_store_search", "acreate_skill", "acreate_interaction", + "acreate_realtime_client_secret", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_calls", + "arealtime_translation_calls", ] ], ) diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index b31db62589d..fef7dbc182b 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,4 +1,3 @@ -import json from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -7,6 +6,31 @@ from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_conte from litellm.types.realtime import RealtimeQueryParams + +from openai import AsyncOpenAI, omit + + +class DummySDKConnectionManager: + def __init__(self, connection): + self.connection = connection + + async def __aenter__(self): + return self.connection + + async def __aexit__(self, exc_type, exc, tb): + return None + + +def make_realtime_sdk_client(): + connection = MagicMock() + connection.send_raw = AsyncMock() + connection.recv_bytes = AsyncMock() + connection.close = AsyncMock() + client = MagicMock(spec=AsyncOpenAI) + client.realtime.connect = MagicMock(return_value=DummySDKConnectionManager(connection)) + return client + + @pytest.mark.parametrize("api_base", ["https://api.openai.com/v1", "https://api.openai.com"]) def test_openai_realtime_handler_url_construction(api_base): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -99,27 +123,10 @@ async def test_async_realtime_success(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -131,6 +138,7 @@ async def test_async_realtime_success(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) mock_realtime_streaming.assert_called_once() @@ -154,28 +162,10 @@ async def test_async_realtime_url_contains_model(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -187,30 +177,48 @@ async def test_async_realtime_url_contains_model(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called with the correct URL - mock_ws_connect.assert_called_once() - called_url = mock_ws_connect.call_args[0][0] - - # Verify the URL contains the model parameter - assert called_url.startswith("wss://api.openai.com/v1/realtime?") - assert f"model={model}" in called_url - - # Verify proper headers were set (GA default: no OpenAI-Beta unless client sent it) - called_kwargs = mock_ws_connect.call_args[1] - assert "additional_headers" in called_kwargs - additional_headers = called_kwargs["additional_headers"] + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] == model + additional_headers = called_kwargs["extra_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert "OpenAI-Beta" not in additional_headers - # Verify SSL is configured (should be an SSLContext or True, not None or False) - assert called_kwargs["ssl"] is not None - assert called_kwargs["ssl"] is not False + assert called_kwargs["max_retries"] == 0 mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() +@pytest.mark.asyncio +async def test_async_realtime_transcription_omits_sdk_model_query(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + websocket = AsyncMock() + logging_obj = MagicMock() + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: + mock_realtime_streaming.return_value.bidirectional_forward = AsyncMock() + + await handler.async_realtime( + model="gpt-live-transcribe", + websocket=websocket, + logging_obj=logging_obj, + api_key="test-key", + query_params={"intent": "transcription"}, + client=sdk_client, + ) + + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] is omit + assert called_kwargs["extra_query"] == {"intent": "transcription"} + + @pytest.mark.asyncio async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it(): """Upstream WS gets OpenAI-Beta: realtime=v1 only when the client WebSocket included it.""" @@ -230,26 +238,10 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() ] } dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -261,11 +253,12 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - mock_ws_connect.assert_called_once() - called_kwargs = mock_ws_connect.call_args[1] - additional_headers = called_kwargs["additional_headers"] + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + additional_headers = called_kwargs["extra_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert additional_headers["OpenAI-Beta"] == "realtime=v1" @@ -289,28 +282,10 @@ async def test_async_realtime_uses_max_size_parameter(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -322,20 +297,14 @@ async def test_async_realtime_uses_max_size_parameter(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called with the max_size parameter - mock_ws_connect.assert_called_once() - called_kwargs = mock_ws_connect.call_args[1] - - # Verify max_size is set (default None for unlimited, matching OpenAI's SDK) - assert "max_size" in called_kwargs - assert called_kwargs["max_size"] is None - # Verify SSL is configured (should be an SSLContext or True, not None or False) - assert called_kwargs["ssl"] is not None - assert called_kwargs["ssl"] is not False - # Default should be None (unlimited) to match OpenAI's official agents SDK - # https://github.com/openai/openai-agents-python/blob/cf1b933660e44fd37b4350c41febab8221801409/src/agents/realtime/openai_realtime.py#L235 + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + connection_options = called_kwargs["websocket_connection_options"] + assert connection_options["max_size"] is REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES + assert "ssl" not in connection_options mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() @@ -360,27 +329,10 @@ async def test_async_realtime_ws_url_has_no_ssl(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -392,19 +344,13 @@ async def test_async_realtime_ws_url_has_no_ssl(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called - mock_ws_connect.assert_called_once() - called_url = mock_ws_connect.call_args[0][0] - called_kwargs = mock_ws_connect.call_args[1] - - # Verify URL was converted from http:// to ws:// - assert called_url.startswith("ws://localhost:8113/v1/realtime?") - assert f"model={model}" in called_url - - # Verify ssl is None for ws:// URLs (the fix for issue #19222) - assert called_kwargs["ssl"] is None + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] == model + assert "ssl" not in called_kwargs["websocket_connection_options"] @pytest.mark.asyncio @@ -526,3 +472,64 @@ async def test_realtime_query_params_preserve_missing_model(monkeypatch): called_kwargs = mock_async_realtime.call_args.kwargs assert called_kwargs["query_params"] == {"intent": "transcription"} + +def test_translation_url_uses_dedicated_path(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + url = handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + assert url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + + +@pytest.mark.asyncio +async def test_translation_websocket_uses_direct_transport(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + backend = AsyncMock() + + class TranslationConnectionManager: + async def __aenter__(self): + return backend + + async def __aexit__(self, exc_type, exc, tb): + return None + + websocket = MagicMock() + websocket.scope = {"headers": []} + websocket.close = AsyncMock() + logging_obj = MagicMock() + handler = OpenAIRealtime() + expected_url = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert ( + handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + == expected_url + ) + + with ( + patch("websockets.connect", return_value=TranslationConnectionManager()) as connect, + patch( # test-quality-ok: transport test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as streaming, + ): + streaming.return_value.bidirectional_forward = AsyncMock() + await handler.async_realtime( + model="gpt-realtime-translate", + websocket=websocket, + logging_obj=logging_obj, + api_base="https://api.openai.com/v1", + api_key="sk-test", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + + connect.assert_called_once() + assert connect.call_args.args[0] == expected_url + assert streaming.call_args.kwargs["translation_session"] is True diff --git a/tests/unit/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py index 3bb9a4ab1ab..27ffcf26878 100644 --- a/tests/unit/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/unit/llms/openai/realtime/test_transcription_sessions.py @@ -21,9 +21,7 @@ from litellm.types.realtime import RealtimeTranscriptionSessionRequest def test_openai_transcription_session_url(): cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -32,9 +30,7 @@ def test_openai_transcription_session_url_strips_trailing_v1(): """A /v1 suffix must not be duplicated in the path.""" cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com/v1", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com/v1", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -46,9 +42,18 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version(): model="whisper-deploy", api_version="2025-04-01-preview", ) - assert ( - url - == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + assert url == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + + +@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) +def test_azure_ga_realtime_http_urls(api_version): + cfg = AzureRealtimeHTTPConfig() + base = "https://my.openai.azure.com" + + assert cfg.get_complete_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/calls") + assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == ( + f"{base}/openai/v1/realtime/transcription_sessions" ) @@ -141,6 +146,30 @@ async def test_client_secret_handler_still_targets_client_secrets_url(): assert kwargs["url"] == "https://api.openai.com/v1/realtime/client_secrets" +@pytest.mark.asyncio +async def test_azure_client_secret_prefers_provider_qualified_routing_model(): + import litellm + + mock_response = MagicMock(spec=httpx.Response) + mock_client = MagicMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=mock_response) + + result = await litellm.acreate_realtime_client_secret( + model="azure/gpt-realtime-2.1", + session={"type": "realtime", "model": "gpt-realtime-2.1"}, + api_base="https://my.openai.azure.com", + api_key="azure-test-key", + api_version="v1", + client=mock_client, + ) + + assert result is mock_response + request = mock_client.post.call_args.kwargs + assert request["url"] == "https://my.openai.azure.com/openai/v1/realtime/client_secrets" + assert request["headers"]["api-key"] == "azure-test-key" + assert request["json"]["session"]["model"] == "gpt-realtime-2.1" + + @pytest.mark.asyncio async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): """ @@ -169,18 +198,14 @@ async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): assert kwargs["url"].endswith("/v1/realtime/transcription_sessions") # The litellm-only routing hint must not be forwarded upstream. assert "model" not in kwargs["json"] - assert kwargs["json"]["input_audio_transcription"] == { - "model": "gpt-realtime-whisper" - } + assert kwargs["json"]["input_audio_transcription"] == {"model": "gpt-realtime-whisper"} def test_append_query_params_skips_existing_keys(): from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler url = "wss://example.com/v1/realtime?model=gpt-4o" - result = BaseLLMHTTPHandler._append_query_params( - url, {"model": "ignored", "intent": "transcription"} - ) + result = BaseLLMHTTPHandler._append_query_params(url, {"model": "ignored", "intent": "transcription"}) assert "model=ignored" not in result assert "intent=transcription" in result diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py new file mode 100644 index 00000000000..6eb2a618b93 --- /dev/null +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -0,0 +1,265 @@ +import gzip +import json +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncOpenAI + +from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig +from litellm.types.realtime import RealtimeSessionConfig + + +def test_realtime_session_config_supports_translation_and_live_transcription_fields(): + session = RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio={ + "input": { + "transcription": { + "model": "gpt-live-transcribe", + "delay": "minimal", + "languages": ["en", "fr"], + "keywords": ["LiteLLM"], + } + }, + "output": {"language": "es"}, + }, + ) + + assert session.audio is not None + assert session.audio.input is not None + assert session.audio.input.transcription is not None + assert session.audio.input.transcription.delay == "minimal" + assert session.audio.input.transcription.languages == ["en", "fr"] + assert session.audio.input.transcription.keywords == ["LiteLLM"] + assert session.audio.output is not None + assert session.audio.output.language == "es" + + +@pytest.mark.parametrize( + "api_base,expected", + [ + ( + "https://api.openai.com", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ( + "https://api.openai.com/v1", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ], +) +def test_openai_translation_client_secret_url(api_base: str, expected: str): + config = OpenAIRealtimeHTTPConfig() + assert config.get_translation_client_secret_url(api_base, "gpt-realtime-translate") == expected + + +def test_openai_translation_calls_url(): + config = OpenAIRealtimeHTTPConfig() + assert ( + config.get_translation_calls_url("https://api.openai.com/v1", "gpt-realtime-translate") + == "https://api.openai.com/v1/realtime/translations/calls" + ) + + +def test_azure_translation_urls_use_ga_paths(): + config = AzureRealtimeHTTPConfig() + assert ( + config.get_translation_client_secret_url("https://example.openai.azure.com", "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets" + ) + assert ( + config.get_translation_calls_url("https://example.openai.azure.com", "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/calls" + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_custom_translation_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/client_secrets"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + request_data = {"session": {"type": "translation", "model": "gpt-realtime-translate"}} + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://api.openai.com", + api_key="sk-test", + request_data=request_data, + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + ) + + assert response.status_code == 200 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/client_secrets" + assert call["json"] == request_data + + +@pytest.mark.asyncio +async def test_azure_translation_client_secret_supports_entra_bearer_auth(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request( + "POST", + "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets", + ), + ) + ) + handler = BaseLLMHTTPHandler() + + await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.openai.azure.com", + api_key="", + request_data={"session": {"type": "translation", "model": "translate-deployment"}}, + logging_obj=MagicMock(), + timeout=10, + provider_config=AzureRealtimeHTTPConfig(), + model="translate-deployment", + extra_headers={"Authorization": "Bearer entra-token"}, + client=client, + ) + + headers = client.post.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer entra-token" + assert "api-key" not in headers + + +@pytest.mark.asyncio +async def test_translation_calls_use_translation_session_and_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 201, + content=b"v=0\r\n", + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/calls"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://api.openai.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + translation=True, + ) + + assert response.status_code == 201 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/calls" + assert call["headers"]["Content-Type"] == "application/sdp" + assert call["content"] == "v=0\r\n" + + +@pytest.mark.asyncio +async def test_standard_client_secret_uses_openai_sdk_resource(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/client_secrets" + return httpx.Response( + 200, + content=gzip.compress(b'{"value":"ek_test"}'), + headers={"content-encoding": "gzip"}, + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + logging_obj = MagicMock() + + response = await handler.async_realtime_client_secret_handler( + api_base="https://example.com", + api_key="sk-test", + request_data={"session": {"type": "realtime", "model": "gpt-realtime-2.1"}}, + logging_obj=logging_obj, + timeout=10, + client=openai_client, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 200 + assert response.json() == {"value": "ek_test"} + assert "content-encoding" not in response.headers + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_openai_sdk_custom_post(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/translations/client_secrets" + assert json.loads(request.content)["session"]["audio"]["output"]["language"] == "es" + return httpx.Response(200, json={"value": "ek_translation"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.com", + api_key="sk-test", + request_data={ + "session": { + "model": "gpt-realtime-translate", + "audio": {"output": {"language": "es"}}, + } + }, + logging_obj=MagicMock(), + timeout=10, + client=openai_client, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 200 + assert response.json() == {"value": "ek_translation"} + + +@pytest.mark.asyncio +async def test_translation_calls_use_openai_sdk_custom_post(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/translations/calls" + body = await request.aread() + assert request.headers["content-type"] == "application/sdp" + assert body == b"v=0\r\n" + return httpx.Response(201, content=b"v=0\r\n") + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="ek_test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://example.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=MagicMock(), + timeout=10, + model="gpt-realtime-translate", + client=openai_client, + translation=True, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 201 + assert response.text == "v=0\r\n" diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py new file mode 100644 index 00000000000..6ccbeaf690a --- /dev/null +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -0,0 +1,297 @@ +import io +import json +import wave +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncOpenAI, AsyncStream, AzureOpenAI + +import litellm +from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, +) +from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription +from litellm.main import _validate_gpt_transcription_request +from litellm.types.utils import TranscriptionResponse +from litellm.utils import get_optional_params_transcription + + +def test_gpt_transcribe_config_uses_native_parameters_and_json(): + config = OpenAIGPTTranscribeAudioTranscriptionConfig() + supported = config.get_supported_openai_params("gpt-transcribe") + assert supported == ["prompt", "response_format", "keywords", "languages", "stream"] + + audio_file = io.BytesIO(b"audio") + request = config.transform_audio_transcription_request( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True}, + litellm_params={}, + ) + assert request.data["response_format"] == "json" + assert request.data["keywords"] == ["LiteLLM"] + assert request.data["languages"] == ["en", "fr"] + assert request.data["stream"] is True + + +def test_gpt_transcribe_optional_params_are_preserved(): + params = get_optional_params_transcription( + model="gpt-transcribe", + custom_llm_provider="openai", + keywords=["LiteLLM", "Realtime API"], + languages=["en", "fr"], + stream=True, + ) + assert params == { + "keywords": ["LiteLLM", "Realtime API"], + "languages": ["en", "fr"], + "stream": True, + } + + +def test_transcription_response_preserves_empty_languages(): + response = TranscriptionResponse(text="hello", languages=[]) + assert response.model_dump()["languages"] == [] + + +@pytest.mark.asyncio +async def test_openai_handler_returns_native_typed_stream(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + assert b'name="stream"' in body + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "languages": [], + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + handler = OpenAIAudioTranscription() + result = handler.audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"], "stream": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + logging_obj=logging_obj, + api_key="sk-test", + api_base="https://example.com/v1", + client=openai_client, + atranscription=True, + provider_config=OpenAIGPTTranscribeAudioTranscriptionConfig(), + ) + stream = await result + assert isinstance(stream, AsyncStream) + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now()) + received = [event async for event in wrapped_stream] + await wrapped_stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + assert received[-1].languages == [] + logging_obj.async_success_handler.assert_awaited_once() + logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + assert logged_response.text == "hello world" + assert logged_response.languages == [] + + +@pytest.mark.asyncio +async def test_atranscription_stream_preserves_duration_for_callback_cost(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + audio_file = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + + stream = await litellm.atranscription( + model="openai/gpt-transcribe", + file=audio_file, + stream=True, + client=openai_client, + litellm_logging_obj=logging_obj, + ) + received = [event async for event in stream] + await stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + logging_obj.async_success_handler.assert_awaited_once() + logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) + + +def test_gpt_transcribe_rejects_conflicting_language_inputs(): + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="cannot be used together"): + litellm.transcription( + model="gpt-transcribe", + file=audio_file, + language="en", + languages=["fr"], + api_key="sk-test", + ) + + +def test_gpt_transcribe_rejects_whisper_response_formats(): + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format='json'"): + litellm.transcription( + model="gpt-transcribe", + file=audio_file, + response_format="verbose_json", + api_key="sk-test", + ) + + +def test_gpt_live_transcribe_rejects_file_transcription(): + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="Realtime API"): + litellm.transcription( + model="gpt-live-transcribe", + file=audio_file, + api_key="sk-test", + ) + + +def test_azure_async_gpt_transcribe_forwards_v1_api_version(): + handler = AzureAudioTranscription() + handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) + + handler.audio_transcriptions( + model="gpt-transcribe", + audio_file=io.BytesIO(b"audio"), + optional_params={"stream": True}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="sk-test", + api_base="https://example.openai.azure.com", + api_version="v1", + atranscription=True, + ) + + assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == "v1" + + +@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) +def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: str | None): + resolved_api_version = _validate_gpt_transcription_request( + model="gpt-transcribe", + custom_llm_provider="azure", + language=None, + languages=None, + response_format="json", + api_version=api_version, + ) + + assert resolved_api_version == litellm.AZURE_DEFAULT_API_VERSION + + +def test_azure_gpt_transcribe_uses_deployment_scoped_route(): + def send_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ( + "https://example.openai.azure.com/openai/deployments/gpt-transcribe/audio/transcriptions" + f"?api-version={litellm.AZURE_DEFAULT_API_VERSION}" + ) + return httpx.Response( + 200, + json={"text": "hello", "languages": [{"code": "en"}], "usage": {"type": "duration", "seconds": 1}}, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(send_response)) + client = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert response.languages is not None + assert [language.code for language in response.languages] == ["en"] + client.close() + + +def test_azure_gpt_transcribe_preserves_dated_api_version(): + resolved_api_version = _validate_gpt_transcription_request( + model="gpt-transcribe", + custom_llm_provider="azure", + language=None, + languages=None, + response_format="json", + api_version="2025-04-01-preview", + ) + + assert resolved_api_version == "2025-04-01-preview" diff --git a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py index 703fa13cbc9..a15ba6d7543 100644 --- a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py +++ b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py @@ -9,6 +9,9 @@ TranscriptionVerbose/Diarized type. from unittest.mock import patch +import pytest + +import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, @@ -55,10 +58,7 @@ class TestDiarizedJsonUsageParsing: assert result.usage.seconds == 295.8 def test_usage_duration_object_accepts_float_seconds(self): - assert ( - TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds - == 295.8 - ) + assert TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds == 295.8 class TestTranscriptionDurationNotInResponseBody: @@ -126,6 +126,60 @@ class TestTranscriptionDurationNotInResponseBody: class TestCostCalculatorReadsDurationFromHiddenParams: """The cost calculator should read duration from _hidden_params via completion_cost().""" + @patch( # test-quality-ok: isolates duration selection from the model pricing table + "litellm.cost_calculator.openai_cost_per_second" + ) + def test_completion_cost_prefers_provider_usage_duration(self, mock_cost_fn): + mock_cost_fn.return_value = (0.001, 0.0) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = { + "audio_transcription_duration": 17.5, + "custom_llm_provider": "openai", + } + + completion_cost( + completion_response=response, + model="gpt-transcribe", + call_type="atranscription", + ) + + mock_cost_fn.assert_called_once() + _, kwargs = mock_cost_fn.call_args + assert kwargs["duration"] == 60.0 + + @pytest.mark.parametrize( + ("model", "provider"), + [ + ("gpt-transcribe", "openai"), + ("azure/gpt-transcribe", "azure"), + ], + ) + def test_gpt_transcribe_provider_usage_duration_costs_without_local_decode( + self, + monkeypatch, + model: str, + provider: str, + ): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = {"custom_llm_provider": provider} + + actual = completion_cost( + completion_response=response, + model=model, + call_type="atranscription", + custom_llm_provider=provider, + ) + + assert actual == pytest.approx(0.0045) + @patch("litellm.cost_calculator.openai_cost_per_second") def test_completion_cost_uses_hidden_params_duration(self, mock_cost_fn): """ diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 6de54aa4af6..622350579a1 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -783,6 +783,37 @@ def test_realtime_webrtc_http_routes_classified_as_llm_api(route): assert RouteChecks.is_management_route(route=route) is False +@pytest.mark.parametrize( + "route", + [ + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", + ], +) +def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route): + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["openai_routes"], + ) + + assert RouteChecks.is_llm_api_route(route=route) is True + assert RouteChecks.is_management_route(route=route) is False + assert ( + RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=valid_token, + ) + is True + ) + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index de76c7257cf..a756454405e 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -119,9 +119,7 @@ def patched_transcription(monkeypatch): return data monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - monkeypatch.setattr( - proxy_server, "check_file_size_under_limit", lambda **kwargs: True - ) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", lambda **kwargs: True) async def _form_data(request): from starlette.datastructures import FormData, UploadFile @@ -153,6 +151,47 @@ def patched_transcription_error(monkeypatch, patched_transcription): yield +@pytest.fixture +def patched_transcription_stream(monkeypatch, patched_transcription): + class _FakeEvent: + def model_dump_json(self): + return '{"type":"transcript.text.done","text":"hello world"}' + + class _FakeAsyncStream: + def __init__(self): + self.closed = False + + def __aiter__(self): + async def _events(): + yield _FakeEvent() + + return _events() + + async def aclose(self): + self.closed = True + + async def _form_data(request): + from starlette.datastructures import FormData, UploadFile + + upload = UploadFile( + filename="audio.mp3", + file=io.BytesIO(b"\x00\x01\x02"), + ) + return FormData([("file", upload), ("model", "gpt-transcribe"), ("stream", "true")]) + + stream = _FakeAsyncStream() + + async def _llm_call(): + return stream + + async def _fake_route_request(*args, **kwargs): + return _llm_call() + + monkeypatch.setattr(proxy_server, "get_form_data", _form_data) + monkeypatch.setattr(proxy_server, "route_request", _fake_route_request) + yield stream + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_happy_path(client, auth_as, patched_speech, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (happy).""" @@ -254,3 +293,15 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error, response = client.post(path, files=files, data=data) assert response.status_code == 500 assert len(response.content) > 0 + + +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcription_stream, path): + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + data = {"model": "gpt-transcribe", "stream": "true"} + with auth_as(): + response = client.post(path, files=files, data=data) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\n' + assert patched_transcription_stream.closed is True diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index f5c97142dde..10c2543021e 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -15,16 +15,25 @@ import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient - -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ConfigGeneralSettings, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) from litellm.proxy.realtime_endpoints.endpoints import ( + _ALLOWED_SESSION_TYPES, + _coerce_realtime_session_type, _decode_realtime_token_payload, _encode_realtime_token_payload, + _prepare_client_secret_session, +) +from litellm.types.realtime import ( + RealtimeAudioInputConfig, + RealtimeAudioTranscriptionConfig, + RealtimeClientSecretRequest, + RealtimeSessionAudioConfig, + RealtimeSessionConfig, ) # --- Unit tests: token encode/decode helpers --- @@ -117,18 +126,107 @@ def proxy_app(monkeypatch): from litellm.proxy import proxy_server monkeypatch.setattr(proxy_server, "master_key", "sk-test-master-key") + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", True) return proxy_server.app +def test_non_billable_realtime_protocols_default_to_disabled(): + assert ConfigGeneralSettings().allow_non_billable_realtime_protocols is False + + +@pytest.mark.parametrize( + ("path", "body"), + ( + ( + "/v1/realtime/client_secrets", + {"model": "gpt-realtime-2"}, + ), + ( + "/v1/realtime/translations/client_secrets", + { + "model": "gpt-realtime-translate", + "session": {"type": "translation", "model": "gpt-realtime-translate"}, + }, + ), + ( + "/v1/realtime/transcription_sessions", + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + ), + ), +) +def test_non_billable_realtime_credential_endpoints_require_opt_in( + proxy_app, + monkeypatch, + path, + body, +): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") + try: + client = TestClient(proxy_app, raise_server_exceptions=False) + with patch( # test-quality-ok: endpoint gate must prove routing is never reached + "litellm.proxy.proxy_server.route_request" + ) as mock_route_request: + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json=body, + ) + + assert response.status_code == 403 + assert "bypasses LiteLLM billing" in response.json()["detail"] + mock_route_request.assert_not_called() + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + ( + "/v1/realtime/calls", + "/v1/realtime/translations/calls", + ), +) +def test_non_billable_realtime_sdp_endpoints_require_opt_in( + proxy_app, + monkeypatch, + path, +): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) + token_payload = _encode_realtime_token_payload( + ephemeral_key="epk", + model_id="gpt-realtime-2", + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + ) + encrypted_token = encrypt_value_helper(token_payload) + client = TestClient(proxy_app, raise_server_exceptions=False) + with patch( # test-quality-ok: endpoint gate must prove routing is never reached + "litellm.proxy.proxy_server.route_request" + ) as mock_route_request: + response = client.post( + path, + headers={"Authorization": f"Bearer {encrypted_token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 403 + assert "bypasses LiteLLM billing" in response.json()["detail"] + mock_route_request.assert_not_called() + + @pytest.fixture def mock_route_request_client_secrets(): """Mock route_request to return a fake upstream client_secrets response.""" future_expires_at = int(time.time()) + 3600 mock_resp = MagicMock(spec=httpx.Response) mock_resp.status_code = 200 - mock_resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) + mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode() mock_resp.headers = {} mock_resp.json.return_value = { @@ -215,9 +313,7 @@ async def test_client_secrets_success_with_mock( mock_pre_call_hook, ): """POST /v1/realtime/client_secrets returns 200 with valid auth and mocked upstream.""" - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -275,13 +371,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -312,12 +402,8 @@ async def test_client_secrets_transcription_routes_on_nested_model( async def _inner(): resp = MagicMock(spec=httpx.Response) resp.status_code = 200 - resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) - resp.content = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ).encode() + resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' + resp.content = (f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}').encode() resp.headers = {} resp.json.return_value = { "value": "upstream_ephemeral_key", @@ -351,13 +437,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -367,10 +447,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( session = captured["data"]["session"] assert session["type"] == "transcription" assert "model" not in session - assert ( - session["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert session["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" encrypted_value = response.json()["value"] decoded = _decode_realtime_token_payload( decrypt_value_helper( @@ -530,10 +607,7 @@ async def test_realtime_calls_replays_transcription_session_type( ) assert captured["session"]["type"] == "transcription" - assert ( - captured["session"]["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert captured["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" # --- transcription_sessions endpoint --- @@ -605,9 +679,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -651,9 +723,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -696,9 +766,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -751,9 +819,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -786,6 +852,20 @@ async def test_realtime_transcription_websocket_default_model_checks_key_scope() assert "is not available for this API key" in close_kwargs["reason"] +def test_realtime_transcription_upstream_query_omits_model(): + from litellm.proxy import proxy_server + + assert ( + proxy_server._resolve_realtime_upstream_query_model( + model="gpt-live-transcribe", + intent="transcription", + is_translation=False, + route_model="gpt-live-transcribe", + ) + is None + ) + + @pytest.mark.asyncio async def test_realtime_transcription_websocket_default_model_checks_team_scope(): from litellm.proxy import proxy_server @@ -947,9 +1027,7 @@ async def test_transcription_sessions_encrypts_client_secret( POST /v1/realtime/transcription_sessions returns 200 and the ephemeral key under client_secret.value must be encrypted (never the raw upstream key). """ - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") captured_route_type = {} async def _capturing_route(*args, **kwargs): @@ -993,16 +1071,12 @@ async def test_transcription_sessions_encrypts_client_secret( assert decrypted is not None assert "upstream_ephemeral_key" in decrypted # Routed through the dedicated transcription_sessions route type. - assert ( - captured_route_type["route_type"] - == "acreate_realtime_transcription_session" - ) + assert captured_route_type["route_type"] == "acreate_realtime_transcription_session" finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) def test_session_type_coerced_for_unknown_value(): - """An unrecognized session_type in the token falls back to 'realtime'.""" payload = _encode_realtime_token_payload( ephemeral_key="epk", model_id="gpt-4o", @@ -1011,12 +1085,222 @@ def test_session_type_coerced_for_unknown_value(): expires_at=None, session_type="INJECTED_TYPE", ) - # Force-deserialize and check the coercion that happens in proxy_realtime_calls. decoded = json.loads(payload) - session_type = decoded.get("session_type") or "realtime" - if session_type not in ("realtime", "transcription"): - session_type = "realtime" - assert session_type == "realtime" + assert decoded["session_type"] == "INJECTED_TYPE" + assert _coerce_realtime_session_type("INJECTED_TYPE") == "realtime" + assert _coerce_realtime_session_type(None) == "realtime" + for allowed_session_type in _ALLOWED_SESSION_TYPES: + assert _coerce_realtime_session_type(allowed_session_type) == allowed_session_type + + +@pytest.mark.asyncio +async def test_translation_client_secret_rejects_disallowed_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-live-transcribe"), + ) + ), + ), + ) + + with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_binds_authorized_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-realtime-whisper"), + ) + ), + ), + ) + + model, session_data, session_type = await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + assert model == "gpt-realtime-translate" + assert session_type == "translation" + assert session_data is not None + assert session_data["model"] == "gpt-realtime-translate" + assert session_data["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/client_secrets", + "/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + ], +) +def test_translation_client_secret_aliases_bind_token_family( + proxy_app, + mock_route_request_client_secrets, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-realtime-translate"], + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_client_secrets(*args, **kwargs) + + try: + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-realtime-translate"}, + ) + + assert response.status_code == 200 + assert captured["route_type"] == "acreate_realtime_translation_client_secret" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + decrypted = decrypt_value_helper( + response.json()["value"], + key="client_secret.value", + exception_type="debug", + ) + decoded = _decode_realtime_token_payload(decrypted or "") + assert decoded is not None + assert decoded["session_type"] == "translation" + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/calls", + "/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", + ], +) +def test_translation_calls_aliases_route_translation_session( + proxy_app, + mock_route_request_realtime_calls, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id="gpt-realtime-translate", + user_id="test-user", + team_id=None, + expires_at=int(time.time()) + 3600, + session_type="translation", + ) + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_realtime_calls(*args, **kwargs) + + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 201 + assert captured["route_type"] == "arealtime_translation_calls" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + + +@pytest.mark.parametrize( + "session_type,path", + [ + ("realtime", "/v1/realtime/translations/calls"), + ("translation", "/v1/realtime/calls"), + ], +) +def test_realtime_calls_reject_cross_family_token(proxy_app, session_type, path): + model = "gpt-realtime-translate" if session_type == "translation" else "gpt-realtime-2" + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id=model, + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + session_type=session_type, + ) + ) + response = TestClient(proxy_app).post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + assert response.status_code == 401 + assert response.json()["error"] == "Token is not valid for this Realtime endpoint" @pytest.mark.asyncio @@ -1142,9 +1426,7 @@ async def test_transcription_sessions_returns_upstream_error_verbatim( return _inner() - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -1184,9 +1466,7 @@ async def test_transcription_sessions_wraps_route_exception( async def _raise_http(*args, **kwargs): raise HTTPException(status_code=403, detail="Model not allowed") - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") try: client = TestClient(proxy_app, raise_server_exceptions=False) with ( diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 14196d4a1ac..fc5a8f9dab4 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -24,6 +24,7 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient +from starlette.datastructures import URL import litellm import litellm.proxy.proxy_server as proxy_server_module @@ -10771,7 +10772,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): def test_realtime_websocket_route_aliases_registered(): - """Realtime sessions reach the proxy via three path aliases stacked on + """Realtime sessions reach the proxy via six path aliases stacked on `realtime_websocket_endpoint`. Dropping any of them silently 405s WebSocket upgrades because the catch-all `/openai/{endpoint:path}` HTTP passthrough only declares HTTP methods. The aliases must also be @@ -10787,7 +10788,14 @@ def test_realtime_websocket_route_aliases_registered(): websocket_paths = {route.path for route in app.routes if isinstance(route, WebSocketRoute)} openai_routes = LiteLLMRoutes.openai_routes.value - for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + for expected in ( + "/openai/v1/realtime", + "/v1/realtime", + "/realtime", + "/openai/v1/realtime/translations", + "/v1/realtime/translations", + "/realtime/translations", + ): assert expected in websocket_paths, ( f"{expected!r} missing from registered WebSocket routes; the " f"realtime endpoint will 405 for clients hitting this path." @@ -10806,7 +10814,7 @@ def _lit6973_fake_realtime_ws() -> MagicMock: ws = MagicMock() ws.headers = {} ws.scope = {"headers": [], "type": "websocket"} - ws.url = "ws://testserver/v1/realtime" + ws.url = URL("ws://testserver/v1/realtime") ws.accept = AsyncMock() ws.send_text = AsyncMock() ws.close = AsyncMock() diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index 7445070bd3a..d1d4a1ae591 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -673,6 +673,8 @@ def test_every_openai_sdk_response_error_code_has_explicit_status_mapping(): ("insufficient_quota", 429), ("vector_store_timeout", 504), ("invalid_prompt", 400), + ("data_residency_mismatch", 400), + ("bio_policy", 400), ("invalid_image", 400), ("invalid_image_format", 400), ("invalid_base64_image", 400), diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cdff03af3b4..b0d2546774e 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1222,6 +1222,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/images/generations", "/v1/realtime", "/v1/realtime/transcription_sessions", + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls", "/v1/images/variations", "/v1/images/edits", "/v1/batch", From d9f21872e26012eefcc09952900ce3225af50a5f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 12:40:44 -0500 Subject: [PATCH 02/45] fix(transcription): type SDK stream duration metadata explicitly --- litellm/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 94aaa1383b0..34b6cfb5c13 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8218,7 +8218,7 @@ async def atranscription( if file is not None: calculated_duration = calculate_request_duration(file) if calculated_duration is not None: - setattr(response, "_litellm_audio_duration", calculated_duration) + response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field return response if not isinstance(response, TranscriptionResponse): raise ValueError( @@ -8562,7 +8562,7 @@ def transcription( calculated_duration: Final = calculate_request_duration(file) if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - setattr(response, "_litellm_audio_duration", calculated_duration) + response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field elif getattr(response, "duration", None) is None and calculated_duration is not None: response.set_audio_transcription_duration(calculated_duration) From 771fd03595a0f5083eb0a0db1e05bca90c3dc03d Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:03:12 -0500 Subject: [PATCH 03/45] fix(realtime): preserve transport settings and translation duration --- .../litellm_core_utils/realtime_streaming.py | 6 ++- litellm/llms/custom_httpx/llm_http_handler.py | 39 ++++++++++--------- litellm/llms/openai/realtime/handler.py | 7 ++-- .../test_realtime_streaming.py | 5 ++- .../realtime/test_openai_realtime_handler.py | 21 +++++++++- .../llms/openai/realtime/test_translation.py | 8 +++- 6 files changed, 58 insertions(+), 28 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 824829cd926..a5c78ca19fa 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -562,7 +562,11 @@ class RealTimeStreaming: if not self._is_translation_session: return self._capture_translation_output_format(event_obj) - if event_obj.get("type") != "session.output_audio.delta": + if event_obj.get("type") not in ( + "session.output_audio.delta", + "response.output_audio.delta", + "response.audio.delta", + ): return delta: Final = event_obj.get("delta") if not isinstance(delta, str): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b6233899abd..c0ce8c08e96 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6892,17 +6892,7 @@ class BaseLLMHTTPHandler: cast_to=httpx.Response, body=request_data, ) - response_headers: Final = { # mutable-ok: httpx requires a concrete response-header mapping - key: value # mutable-ok: transport headers are materialized after filtering - for key, value in raw_response.headers.items() # mutable-ok: transport headers are materialized - if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") - } - return httpx.Response( - status_code=raw_response.status_code, - headers=response_headers, - content=raw_response.content, - request=httpx.Request("POST", f"{normalized_api_base}/realtime/client_secrets"), - ) + return self._decoded_realtime_sdk_response(raw_response) finally: if owns_client: await openai_client.close() @@ -6987,11 +6977,12 @@ class BaseLLMHTTPHandler: key: str(value) for key, value in (extra_headers or {}).items() }, ) - return await configured_client.post( + raw_response: Final = await configured_client.post( "/realtime/translations/client_secrets", cast_to=httpx.Response, body=request_data, ) + return self._decoded_realtime_sdk_response(raw_response) finally: if owns_client: await openai_client.close() @@ -7129,11 +7120,12 @@ class BaseLLMHTTPHandler: }, }, ) - return await configured_client.post( + translation_response: Final = await configured_client.post( "/realtime/translations/calls", cast_to=httpx.Response, content=sdp_text.encode("utf-8"), ) + return self._decoded_realtime_sdk_response(translation_response) realtime_session_data: Final = cast( # cast-ok: endpoint validation produced an OpenAI realtime session RealtimeSessionCreateRequestParam, session_data, @@ -7147,16 +7139,25 @@ class BaseLLMHTTPHandler: extra_headers=sdk_extra_headers, timeout=timeout, ) - return httpx.Response( - status_code=raw_response.status_code, - headers=raw_response.headers, - content=raw_response.content, - request=httpx.Request("POST", f"{normalized_api_base}/realtime/calls"), - ) + return self._decoded_realtime_sdk_response(raw_response.http_response) finally: if owns_client: await openai_client.close() + @staticmethod + def _decoded_realtime_sdk_response(response: httpx.Response) -> httpx.Response: + headers: Final = { # mutable-ok: httpx accepts a concrete response header mapping + key: value + for key, value in response.headers.items() + if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") + } + return httpx.Response( + status_code=response.status_code, + headers=headers, + content=response.content, + request=response.request, + ) + @staticmethod def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: if isinstance(client, AsyncHTTPHandler): diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index ef63afa269b..3babecd3a91 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -196,15 +196,14 @@ class OpenAIRealtime(OpenAIChatCompletion): ) -> AbstractAsyncContextManager[object]: import websockets - if realtime_mode == "translation" or client is None: + if realtime_mode == "translation" or not isinstance(client, AsyncOpenAI): return websockets.connect( url, additional_headers=headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=ssl_config, + **({"open_timeout": timeout} if timeout is not None else {}), ) - if not isinstance(client, AsyncOpenAI): - raise TypeError("client must be an AsyncOpenAI instance") openai_client: Final = client model_query: Final = query_params.get("model") extra_query: Final = { # mutable-ok: OpenAI SDK accepts a mutable query-parameter mapping @@ -217,6 +216,8 @@ class OpenAIRealtime(OpenAIChatCompletion): extra_headers=headers, websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + **({"ssl": ssl_config} if url.startswith("wss://") else {}), + **({"open_timeout": timeout} if timeout is not None else {}), }, max_retries=0, ) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index b62fceeb53e..4901f574769 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3043,7 +3043,8 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] -def test_translation_audio_duration_is_finalized_once(): +@pytest.mark.parametrize("event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"]) +def test_translation_audio_duration_is_finalized_once(event_type: str): import base64 streaming = RealTimeStreaming( @@ -3054,7 +3055,7 @@ def test_translation_audio_duration_is_finalized_once(): translation_session=True, ) payload = base64.b64encode(bytes(48000)).decode() - streaming._capture_translation_output_audio({"type": "session.output_audio.delta", "delta": payload}) + streaming._capture_translation_output_audio({"type": event_type, "delta": payload}) streaming._finalize_translation_usage() streaming._finalize_translation_usage() diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index fef7dbc182b..e1a7f054316 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -304,7 +304,7 @@ async def test_async_realtime_uses_max_size_parameter(): called_kwargs = sdk_client.realtime.connect.call_args.kwargs connection_options = called_kwargs["websocket_connection_options"] assert connection_options["max_size"] is REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES - assert "ssl" not in connection_options + assert connection_options["ssl"] is not None mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() @@ -533,3 +533,22 @@ async def test_translation_websocket_uses_direct_transport(): connect.assert_called_once() assert connect.call_args.args[0] == expected_url assert streaming.call_args.kwargs["translation_session"] is True + + +@pytest.mark.parametrize("sdk_client", [True, False]) +def test_connection_manager_preserves_transport_settings(sdk_client: bool): + import ssl + from litellm.llms.openai.realtime.handler import OpenAIRealtime + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + client = make_realtime_sdk_client() if sdk_client else MagicMock(spec=AsyncHTTPHandler) + ssl_config = ssl.create_default_context() + with patch("websockets.connect") as connect: + OpenAIRealtime()._create_connection_manager( + api_base="https://example.com", api_key="test", model="gpt-realtime-2.1", + query_params={"model": "gpt-realtime-2.1"}, headers={}, timeout=7.0, + realtime_mode="realtime", ssl_config=ssl_config, client=client, url="wss://example.com/v1/realtime", + ) + options = client.realtime.connect.call_args.kwargs["websocket_connection_options"] if sdk_client else connect.call_args.kwargs + assert options["ssl"] is ssl_config + assert options["open_timeout"] == 7.0 diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py index 6eb2a618b93..a329200b2f7 100644 --- a/tests/unit/llms/openai/realtime/test_translation.py +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -209,7 +209,7 @@ async def test_translation_client_secret_uses_openai_sdk_custom_post(): async def send_response(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/realtime/translations/client_secrets" assert json.loads(request.content)["session"]["audio"]["output"]["language"] == "es" - return httpx.Response(200, json={"value": "ek_translation"}) + return httpx.Response(200, content=gzip.compress(b'{"value":"ek_translation"}'), headers={"content-encoding": "gzip"}) http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) @@ -233,6 +233,8 @@ async def test_translation_client_secret_uses_openai_sdk_custom_post(): assert response.status_code == 200 assert response.json() == {"value": "ek_translation"} + assert "content-encoding" not in response.headers + assert int(response.headers["content-length"]) == len(response.content) @pytest.mark.asyncio @@ -242,7 +244,7 @@ async def test_translation_calls_use_openai_sdk_custom_post(): body = await request.aread() assert request.headers["content-type"] == "application/sdp" assert body == b"v=0\r\n" - return httpx.Response(201, content=b"v=0\r\n") + return httpx.Response(201, content=gzip.compress(b"v=0\r\n"), headers={"content-encoding": "gzip"}) http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) openai_client = AsyncOpenAI(api_key="ek_test", base_url="https://example.com/v1", http_client=http_client) @@ -263,3 +265,5 @@ async def test_translation_calls_use_openai_sdk_custom_post(): assert response.status_code == 201 assert response.text == "v=0\r\n" + assert "content-encoding" not in response.headers + assert int(response.headers["content-length"]) == len(response.content) From 98f0d3c7c687103ce024bdda2a2075e7604f1bec Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:10:23 -0500 Subject: [PATCH 04/45] refactor(realtime): reuse SDK client initialization --- litellm/llms/custom_httpx/llm_http_handler.py | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c0ce8c08e96..215b4139a5a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6861,16 +6861,7 @@ class BaseLLMHTTPHandler: header auth when available; falls back to the legacy OpenAI-style defaults. """ if use_openai_sdk: - trimmed_api_base: Final = api_base.rstrip("/") - normalized_api_base: Final = ( - trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" - ) - owns_client: Final = not isinstance(client, AsyncOpenAI) - openai_client: Final = ( - client - if isinstance(client, AsyncOpenAI) - else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) - ) + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) logging_obj.pre_call( input=request_data, api_key="", @@ -6953,15 +6944,7 @@ class BaseLLMHTTPHandler: use_openai_sdk: bool = False, ) -> httpx.Response: if use_openai_sdk: - normalized_api_base = api_base.rstrip("/") - if not normalized_api_base.endswith("/v1"): - normalized_api_base = f"{normalized_api_base}/v1" - owns_client: Final = not isinstance(client, AsyncOpenAI) - openai_client: Final = ( - client - if isinstance(client, AsyncOpenAI) - else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) - ) + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) logging_obj.pre_call( input=request_data, api_key="", @@ -7092,14 +7075,8 @@ class BaseLLMHTTPHandler: client: object | None, translation: bool, ) -> httpx.Response: - normalized_api_base = api_base.rstrip("/") - if not normalized_api_base.endswith("/v1"): - normalized_api_base = f"{normalized_api_base}/v1" - owns_client: Final = not isinstance(client, AsyncOpenAI) - openai_client: Final = ( - client - if isinstance(client, AsyncOpenAI) - else AsyncOpenAI(api_key=openai_ephemeral_key, base_url=normalized_api_base, max_retries=0) + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client( + api_base, openai_ephemeral_key, client ) logging_obj.pre_call( input="realtime_sdp_offer", @@ -7144,6 +7121,14 @@ class BaseLLMHTTPHandler: if owns_client: await openai_client.close() + @staticmethod + def _realtime_sdk_client(api_base: str, api_key: str, client: object | None) -> tuple[AsyncOpenAI, bool, str]: + trimmed_api_base: Final = api_base.rstrip("/") + normalized_api_base: Final = trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" + if isinstance(client, AsyncOpenAI): + return client, False, normalized_api_base + return AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0), True, normalized_api_base + @staticmethod def _decoded_realtime_sdk_response(response: httpx.Response) -> httpx.Response: headers: Final = { # mutable-ok: httpx accepts a concrete response header mapping From 8fa5c0e91484cd776a677d46e9341687ba1d0593 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:31:02 -0500 Subject: [PATCH 05/45] fix(audio): bill finalized streams and authorize transcription models --- litellm/cost_calculator.py | 20 ++++++++++++++++++- litellm/litellm_core_utils/litellm_logging.py | 18 ++++++++++++++--- .../usage_object_transformation.py | 10 +++++++--- litellm/proxy/realtime_endpoints/endpoints.py | 7 +++++++ litellm/utils.py | 2 +- .../transcriptions/test_gpt_transcribe.py | 15 ++++++++------ .../test_realtime_webrtc_endpoints.py | 17 ++++++++++++++-- 7 files changed, 73 insertions(+), 16 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 493924320be..bce51735e7a 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -685,7 +685,25 @@ def cost_per_token( vertex_location=vertex_location, ) elif call_type == "atranscription" or call_type == "transcription": - if _transcription_usage_has_token_details(usage_block): + transcription_model_info: Final = _cached_get_model_info_helper( + model=model_without_prefix, custom_llm_provider=custom_llm_provider + ) + has_token_pricing: Final = any( + transcription_model_info.get(field) + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + ) + ) + if _transcription_usage_has_token_details(usage_block) and ( + has_token_pricing + or ( + transcription_model_info.get("input_cost_per_second") is None + and transcription_model_info.get("output_cost_per_second") is None + ) + ): return generic_cost_per_token( model=model_without_prefix, usage=usage_block, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d75d35e4531..3be9f28f194 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3817,7 +3817,12 @@ class Logging(LiteLLMLoggingBaseClass): if "async_complete_streaming_response" in self.model_call_details: return # break out of this. complete_streaming_response: Final[ - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None ] = self._get_assembled_streaming_response( result=result, start_time=start_time, @@ -4731,10 +4736,17 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: list[object], - ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None: + ) -> ( + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None + ): if self.stream is not True: return None - if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse): + if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)): return result elif isinstance( result, diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index a02c40b7611..a34e4de2dca 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -31,9 +31,13 @@ class TranscriptionUsageObjectTransformation: prompt_tokens=usage_object.input_tokens, completion_tokens=usage_object.output_tokens, total_tokens=usage_object.total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=usage_object.input_token_details.text_tokens, - audio_tokens=usage_object.input_token_details.audio_tokens, + prompt_tokens_details=( + PromptTokensDetailsWrapper( + text_tokens=usage_object.input_token_details.text_tokens, + audio_tokens=usage_object.input_token_details.audio_tokens, + ) + if usage_object.input_token_details is not None + else None ), ) return None diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index e3038c21a6d..897f7a347f4 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -682,6 +682,13 @@ async def create_realtime_transcription_session( ) transcription_session: Final = {k: v for k, v in body.items() if k != "model"} + await _authorize_and_bind_nested_transcription_models( + session_data=transcription_session, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + _set_transcription_model_on_session(session=transcription_session, model=model, create_if_missing=True) data = {"model": model, "transcription_session": transcription_session} data = await add_litellm_data_to_request( diff --git a/litellm/utils.py b/litellm/utils.py index 66505a389f5..4dfd59540de 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4053,7 +4053,7 @@ def get_optional_params_transcription( from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS # retrieve all parameters passed to the function - passed_params: Final = locals() + passed_params: Final = locals().copy() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") passed_params.pop("model") diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 6ccbeaf690a..35bed209400 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -143,10 +143,12 @@ async def test_atranscription_stream_preserves_duration_for_callback_cost(): http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) - logging_obj = MagicMock() - logging_obj.model_call_details = {} - logging_obj.async_success_handler = AsyncMock() - logging_obj.async_failure_handler = AsyncMock() + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-transcribe", messages=[], stream=True, call_type="atranscription", + start_time=datetime.now(), litellm_call_id="transcription-cost-test", function_id="transcription-cost-test", + ) audio_file = io.BytesIO() with wave.open(audio_file, "wb") as wav_file: wav_file.setnchannels(1) @@ -167,8 +169,9 @@ async def test_atranscription_stream_preserves_duration_for_callback_cost(): await openai_client.close() assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] - logging_obj.async_success_handler.assert_awaited_once() - logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + logged_response = logging_obj.model_call_details["async_complete_streaming_response"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.000075) + assert logging_obj.model_call_details["standard_logging_object"]["response_cost"] == pytest.approx(0.000075) assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 10c2543021e..19242d7bdd2 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -661,8 +661,14 @@ def test_transcription_sessions_requires_auth(proxy_app): @pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + {"model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"input_audio_transcription": {"model": "gpt-4o-realtime-preview"}, "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "gpt-realtime-whisper"}}, +]) async def test_transcription_sessions_rejects_disallowed_resolved_model( - proxy_app, + proxy_app, body, ): proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( user_id="test-user", @@ -679,7 +685,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + json=body, ) assert response.status_code == 403 @@ -1032,6 +1038,7 @@ async def test_transcription_sessions_encrypts_client_secret( async def _capturing_route(*args, **kwargs): captured_route_type["route_type"] = kwargs.get("route_type") + captured_route_type["session"] = kwargs["data"]["transcription_session"] return await mock_route_request_transcription_sessions(*args, **kwargs) try: @@ -1054,6 +1061,8 @@ async def test_transcription_sessions_encrypts_client_secret( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, json={ + "model": "gpt-realtime-whisper", + "audio": {"input": {"transcription": {"model": "gpt-live-transcribe", "language": "en"}}}, "input_audio_format": "pcm16", "input_audio_transcription": {"model": "gpt-realtime-whisper"}, }, @@ -1072,6 +1081,10 @@ async def test_transcription_sessions_encrypts_client_secret( assert "upstream_ephemeral_key" in decrypted # Routed through the dedicated transcription_sessions route type. assert captured_route_type["route_type"] == "acreate_realtime_transcription_session" + assert captured_route_type["session"]["input_audio_transcription"]["model"] == "gpt-realtime-whisper" + assert captured_route_type["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", "language": "en", + } finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) From d6518778c124b1703dfbc4795bc70f6d7fc6dfca Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:35:42 -0500 Subject: [PATCH 06/45] refactor(audio): keep transcription pricing selection together --- litellm/cost_calculator.py | 39 +++++++++---------- litellm/litellm_core_utils/litellm_logging.py | 9 +---- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index bce51735e7a..b65bd02a911 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -315,12 +315,29 @@ def _get_additional_costs( return None -def _transcription_usage_has_token_details( +def _transcription_uses_token_pricing( + model: str, + custom_llm_provider: str | None, usage_block: Usage | None, ) -> bool: if usage_block is None: return False + model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + has_token_pricing: Final = any( + model_info.get(field) + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + ) + ) + if not has_token_pricing and ( + model_info.get("input_cost_per_second") is not None or model_info.get("output_cost_per_second") is not None + ): + return False + prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0 completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0 prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None) @@ -685,25 +702,7 @@ def cost_per_token( vertex_location=vertex_location, ) elif call_type == "atranscription" or call_type == "transcription": - transcription_model_info: Final = _cached_get_model_info_helper( - model=model_without_prefix, custom_llm_provider=custom_llm_provider - ) - has_token_pricing: Final = any( - transcription_model_info.get(field) - for field in ( - "input_cost_per_token", - "output_cost_per_token", - "input_cost_per_audio_token", - "output_cost_per_audio_token", - ) - ) - if _transcription_usage_has_token_details(usage_block) and ( - has_token_pricing - or ( - transcription_model_info.get("input_cost_per_second") is None - and transcription_model_info.get("output_cost_per_second") is None - ) - ): + if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block): return generic_cost_per_token( model=model_without_prefix, usage=usage_block, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3be9f28f194..f5d8ebe53e4 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3816,14 +3816,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE if "async_complete_streaming_response" in self.model_call_details: return # break out of this. - complete_streaming_response: Final[ - ModelResponse - | TextCompletionResponse - | ResponsesAPIResponse - | InteractionsAPIResponse - | TranscriptionResponse - | None - ] = self._get_assembled_streaming_response( + complete_streaming_response: Final = self._get_assembled_streaming_response( result=result, start_time=start_time, end_time=end_time, From 3e71406b2956fa110673250a2e222f5b0038b36e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 18:27:12 -0500 Subject: [PATCH 07/45] fix(realtime): reconcile model metadata and test contracts --- litellm/llms/custom_httpx/llm_http_handler.py | 18 +- litellm/llms/openai/realtime/handler.py | 12 +- ...odel_prices_and_context_window_backup.json | 273 ++++++++++++-- model_prices_and_context_window.json | 273 ++++++++++++-- .../llm_cost_calc/test_llm_cost_calc_utils.py | 48 ++- .../realtime/test_azure_realtime_handler.py | 15 + .../realtime/test_openai_realtime_handler.py | 1 + .../test_realtime_webrtc_endpoints.py | 2 +- tests/unit/test_cost_calculator.py | 63 ++++ .../test_model_prices_and_context_window.py | 33 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 354 +++++++++++++++++- 11 files changed, 1007 insertions(+), 85 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 215b4139a5a..dbde287b10a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6956,9 +6956,9 @@ class BaseLLMHTTPHandler: try: configured_client: Final = openai_client.with_options( timeout=timeout, - set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping - key: str(value) for key, value in (extra_headers or {}).items() - }, + set_default_headers=MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ), ) raw_response: Final = await configured_client.post( "/realtime/translations/client_secrets", @@ -7092,9 +7092,9 @@ class BaseLLMHTTPHandler: timeout=timeout, set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping "Content-Type": "application/sdp", - **{ # mutable-ok: caller headers are normalized into the SDK header mapping - key: str(value) for key, value in (extra_headers or {}).items() - }, + **MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ), }, ) translation_response: Final = await configured_client.post( @@ -7107,9 +7107,9 @@ class BaseLLMHTTPHandler: RealtimeSessionCreateRequestParam, session_data, ) - sdk_extra_headers: Final = { # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping - key: str(value) for key, value in (extra_headers or {}).items() - } + sdk_extra_headers: Final = MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ) raw_response: Final = await openai_client.realtime.calls.with_raw_response.create( sdp=sdp_text, session=realtime_session_data, diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 3babecd3a91..1faebd4bb64 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -7,7 +7,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. import ssl from collections.abc import Mapping from contextlib import AbstractAsyncContextManager -from types import TracebackType +from types import MappingProxyType, TracebackType from typing import Any, Final, cast from openai import AsyncOpenAI, omit @@ -202,13 +202,11 @@ class OpenAIRealtime(OpenAIChatCompletion): additional_headers=headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=ssl_config, - **({"open_timeout": timeout} if timeout is not None else {}), + **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), ) openai_client: Final = client model_query: Final = query_params.get("model") - extra_query: Final = { # mutable-ok: OpenAI SDK accepts a mutable query-parameter mapping - key: value for key, value in query_params.items() if key != "model" - } + extra_query: Final = MappingProxyType({key: value for key, value in query_params.items() if key != "model"}) sdk_model: Final = omit if query_params.get("intent") == "transcription" else model_query or model sdk_connection_manager: Final = openai_client.realtime.connect( model=sdk_model, @@ -216,8 +214,8 @@ class OpenAIRealtime(OpenAIChatCompletion): extra_headers=headers, websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - **({"ssl": ssl_config} if url.startswith("wss://") else {}), - **({"open_timeout": timeout} if timeout is not None else {}), + **(MappingProxyType({"ssl": ssl_config}) if url.startswith("wss://") else MappingProxyType({})), + **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), }, max_retries=0, ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb9bfa6ffe4..48c83fb891e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,4 +1,229 @@ { + "azure/gpt-live-transcribe": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-transcribe": { + "input_cost_per_second": 7.5e-05, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/audio/transcriptions", + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -6536,6 +6761,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 3.2e-05, @@ -6565,12 +6791,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 1e-05, @@ -6600,6 +6829,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -29035,7 +29266,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -29143,7 +29373,6 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -29238,7 +29467,6 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -29337,7 +29565,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -29399,7 +29626,6 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -29458,7 +29684,6 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -30064,7 +30289,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -30125,7 +30349,6 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30186,7 +30409,6 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30247,7 +30469,6 @@ }, "vertex_ai/gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -32353,7 +32574,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -32414,7 +32634,6 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32475,7 +32694,6 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32536,7 +32754,6 @@ }, "gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -38226,7 +38443,6 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -38474,7 +38690,6 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -38575,7 +38790,6 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, - "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -38799,6 +39013,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -38836,6 +39052,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -52311,7 +52529,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -52381,7 +52598,6 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -52439,7 +52655,6 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -52538,7 +52753,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52601,7 +52815,6 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52661,7 +52874,6 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -63166,7 +63378,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -63184,7 +63397,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -63210,7 +63424,13 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], "supported_modalities": [ "audio" ], @@ -63219,7 +63439,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb9bfa6ffe4..48c83fb891e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,4 +1,229 @@ { + "azure/gpt-live-transcribe": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-transcribe": { + "input_cost_per_second": 7.5e-05, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/audio/transcriptions", + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -6536,6 +6761,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 3.2e-05, @@ -6565,12 +6791,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 1e-05, @@ -6600,6 +6829,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -29035,7 +29266,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -29143,7 +29373,6 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -29238,7 +29467,6 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -29337,7 +29565,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -29399,7 +29626,6 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -29458,7 +29684,6 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -30064,7 +30289,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -30125,7 +30349,6 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30186,7 +30409,6 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30247,7 +30469,6 @@ }, "vertex_ai/gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -32353,7 +32574,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -32414,7 +32634,6 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32475,7 +32694,6 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32536,7 +32754,6 @@ }, "gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -38226,7 +38443,6 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -38474,7 +38690,6 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -38575,7 +38790,6 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, - "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -38799,6 +39013,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -38836,6 +39052,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -52311,7 +52529,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -52381,7 +52598,6 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -52439,7 +52655,6 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -52538,7 +52753,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52601,7 +52815,6 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52661,7 +52874,6 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -63166,7 +63378,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -63184,7 +63397,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -63210,7 +63424,13 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], "supported_modalities": [ "audio" ], @@ -63219,7 +63439,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index c99147d2e63..2a328123f5a 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -29,6 +29,7 @@ from litellm.llms.gemini.image_generation.cost_calculator import ( from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, @@ -42,6 +43,39 @@ from litellm.types.utils import ( ) +def test_realtime_cached_modality_breakdown_matches_prompt_cost(_local_model_cost_map): + model: Final = "gpt-realtime-2.1-mini" + rates: Final = litellm.model_cost[model] + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=400, + image_tokens=200, + cached_tokens=300, + cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=150, image_tokens=50), + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + breakdown: Final = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) + cached_cost: Final = ( + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + ) + uncached_cost: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + ) + + assert breakdown.cache_read_cost == pytest.approx(cached_cost) + assert prompt_cost == pytest.approx(uncached_cost + cached_cost) + + @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -324,8 +358,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ) - - def test_image_tokens_with_custom_pricing(): """Test that image_tokens in completion are properly costed with output_cost_per_image_token.""" from unittest.mock import patch @@ -2095,6 +2127,10 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_tokens_details: PromptTokensDetailsResult = { "cache_hit_tokens": 0, "cache_hit_audio_tokens": 0, + "cached_text_tokens": 0, + "cached_audio_tokens": 0, + "cached_image_tokens": 0, + "has_cached_tokens_details": False, "cache_creation_tokens": 0, "cache_creation_token_details": CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, @@ -2330,10 +2366,6 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo assert round(cost, 10) == round(expected_cost, 10) - - - - def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): usage = Usage( prompt_tokens=0, @@ -2512,8 +2544,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode assert base == located - - def test_vertex_uplift_invalid_multiplier_defaults_to_one(): """A malformed multiplier in the cost map degrades to base pricing, never raises.""" from litellm.litellm_core_utils.llm_cost_calc.utils import ( @@ -3787,8 +3817,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima assert cost == pytest.approx(0.07) - - @pytest.mark.parametrize( ("custom_llm_provider", "model"), [ diff --git a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py index f1db76c1e21..203727dfaf4 100644 --- a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py @@ -259,6 +259,21 @@ async def test_construct_url_v1_protocol(): assert url.count("/realtime") == 1 +def test_construct_url_translation_protocol(): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + + url = AzureOpenAIRealtime()._construct_url( + api_base="https://my-endpoint.openai.azure.com", + model="translate-deployment", + api_version=None, + realtime_protocol="GA", + query_params={"model": "translate-deployment"}, + realtime_mode="translation", + ) + + assert url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime/translations?model=translate-deployment" + + @pytest.mark.asyncio @pytest.mark.parametrize("protocol", ["ga", "Ga", "gA", "V1", "v1", "GA"]) async def test_construct_url_case_insensitive_protocol(protocol): diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index e1a7f054316..704efe8323a 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,3 +1,4 @@ +import json from unittest.mock import AsyncMock, MagicMock, patch import pytest diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 19242d7bdd2..f9f75c39db4 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -1121,7 +1121,7 @@ async def test_translation_client_secret_rejects_disallowed_nested_transcription ), ) - with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available for this API key"): await _prepare_client_secret_session( req=req, user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 5d4eb5fae12..ef72c61df7e 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10395,3 +10395,66 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry is not None assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) + + +@pytest.mark.parametrize("provider", ("openai", "azure")) +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str): + model_name: Final = f"azure/{model}" if provider == "azure" else model + rates: Final = litellm.model_cost[model_name] + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.created", "session": {"model": model}}, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 300, + "total_tokens": 1300, + "input_token_details": { + "text_tokens": 400, + "audio_tokens": 400, + "image_tokens": 200, + "cached_tokens": 300, + "cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50}, + }, + "output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100}, + } + }, + }, + ] + combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events) + actual: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=combined, + custom_llm_provider=provider, + litellm_model_name=model_name, + ) + expected: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + + 200 * rates["output_cost_per_token"] + + 100 * rates["output_cost_per_audio_token"] + ) + + assert actual == pytest.approx(expected) + + +def test_realtime_translation_duration_cost(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 701b79d0ba9..73a9af91eef 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -791,3 +791,36 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa sibling = rows["volcengine/doubao-seed-2-1-pro-260628"] assert isinstance(sibling, dict) assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")) + + +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["cache_read_input_image_token_cost"] > 0 + assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"] + assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"] + + +@pytest.mark.parametrize( + "model,mode", + ( + ("gpt-realtime-translate", "realtime"), + ("gpt-live-transcribe", "audio_transcription"), + ("gpt-transcribe", "audio_transcription"), + ), +) +def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["mode"] == azure["mode"] == mode + assert azure["supports_audio_input"] is True + assert azure["supported_endpoints"] + + +def test_model_prices_backup_is_synchronized(prices: dict): + backup: Final = json.loads(BACKUP_PRICES_PATH.read_text()) + + assert backup == prices diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bd08b743c7d..8d918949310 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10991,7 +10991,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_3"]; + get: operations["websocket_realtime_websocket_endpoint_get_6"]; put?: never; post?: never; delete?: never; @@ -11058,6 +11058,60 @@ export interface paths { patch?: never; trace?: never; }; + "/openai/v1/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get_3"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/openai/v1/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_openai_v1_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/openai/v1/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai/v1/responses": { parameters: { query?: never; @@ -13965,7 +14019,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get"]; + get: operations["websocket_realtime_websocket_endpoint_get_4"]; put?: never; post?: never; delete?: never; @@ -14032,6 +14086,60 @@ export interface paths { patch?: never; trace?: never; }; + "/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/register": { parameters: { query?: never; @@ -21667,7 +21775,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_2"]; + get: operations["websocket_realtime_websocket_endpoint_get_5"]; put?: never; post?: never; delete?: never; @@ -21734,6 +21842,60 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_v1_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_v1_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/rerank": { parameters: { query?: never; @@ -28465,7 +28627,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -29805,6 +29967,12 @@ export interface components { * @description opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine */ allow_cli_sso_verification_uri_complete?: boolean | null; + /** + * Allow Non Billable Realtime Protocols + * @description Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement + * @default false + */ + allow_non_billable_realtime_protocols: boolean; /** * Allow Unmanaged Response Ids * @description If True, lets keys address Responses API ids that this proxy did not issue (raw provider ids, or ids issued before response-id encryption was configured). Such an id carries no owner, so no ownership check can run on it; ids this proxy did issue keep full ownership enforcement. Off by default, in which case an unrecognized response id is rejected with 403 @@ -66698,6 +66866,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get_3: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_openai_v1_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; responses_api_openai_v1_responses_post: { parameters: { query?: never; @@ -69975,7 +70201,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get: { + websocket_realtime_websocket_endpoint_get_4: { parameters: { query?: never; header?: never; @@ -70053,6 +70279,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; register_client_register_post: { parameters: { query?: { @@ -80877,7 +81161,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_2: { + websocket_realtime_websocket_endpoint_get_5: { parameters: { query?: never; header?: never; @@ -80955,6 +81239,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_v1_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_v1_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; rerank_v1_rerank_post: { parameters: { query?: never; From e0ae1d2925b29a76f39ff4f93b4f97cb243a2c81 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 18:59:55 -0500 Subject: [PATCH 08/45] fix(realtime): preserve provider duration and model capabilities --- cookbook/gpt_realtime_translate.py | 6 ++- .../litellm_core_utils/realtime_streaming.py | 16 +++++++ litellm/main.py | 34 ++++++++++++--- ...odel_prices_and_context_window_backup.json | 6 +++ litellm/types/utils.py | 1 + litellm/utils.py | 3 ++ model_prices_and_context_window.json | 6 +++ .../cookbook/test_gpt_realtime_translate.py | 43 +++++++++++++++++++ .../test_realtime_streaming.py | 29 ++++++++++++- 9 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 tests/unit/cookbook/test_gpt_realtime_translate.py diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py index 73c91eaa1e9..bae89b829ca 100644 --- a/cookbook/gpt_realtime_translate.py +++ b/cookbook/gpt_realtime_translate.py @@ -176,7 +176,11 @@ async def receive_translation( output.setframerate(SAMPLE_RATE) write_stdout("Translation: ", end="", flush=True) while True: - timeout = OUTPUT_IDLE_TIMEOUT_SECONDS if sender_finished.is_set() else INITIAL_RESPONSE_TIMEOUT_SECONDS + timeout = ( + OUTPUT_IDLE_TIMEOUT_SECONDS + if sender_finished.is_set() and audio_received.is_set() + else INITIAL_RESPONSE_TIMEOUT_SECONDS + ) try: raw_event = await asyncio.wait_for(connection.recv(), timeout=timeout) except TimeoutError: diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a5c78ca19fa..17824a281ca 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -561,6 +561,21 @@ class RealTimeStreaming: def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None: if not self._is_translation_session: return + if event_obj.get("type") == "session.closed": + usage: Final = event_obj.get("usage") + output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None + if isinstance(output_seconds, (int, float)): + if not self._should_store_message(event_obj): + self.messages.append( + OpenAIRealtimeTranslationClosedEvent( + type="session.closed", + usage=OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=output_seconds + ), + ) + ) + self._translation_usage_finalized = True + return self._capture_translation_output_format(event_obj) if event_obj.get("type") not in ( "session.output_audio.delta", @@ -1271,6 +1286,7 @@ class RealTimeStreaming: continue if isinstance(event, dict): self._detect_transcription_session_from_backend(event) + self._capture_translation_output_audio(event) is_session_created_event = isinstance(event, dict) and event.get("type") == "session.created" if is_session_created_event: if self._uses_deferred_backend_setup() and not self._backend_setup_complete: diff --git a/litellm/main.py b/litellm/main.py index 34b6cfb5c13..53b63de1e9e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -23,7 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy -from functools import partial +from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -8248,6 +8248,23 @@ async def atranscription( ) +@lru_cache(maxsize=1) +def _bundled_transcription_response_formats() -> Mapping[str, tuple[str, ...]]: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + catalog: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python( + GetModelCostMap.load_local_model_cost_map() + ) + formats_adapter: Final = TypeAdapter(tuple[str, ...]) + return MappingProxyType( + { + model: formats_adapter.validate_python(entry["supported_transcription_response_formats"]) + for model, entry in catalog.items() + if "supported_transcription_response_formats" in entry + } + ) + + def _validate_gpt_transcription_request( model: str, custom_llm_provider: str, @@ -8256,25 +8273,30 @@ def _validate_gpt_transcription_request( response_format: str | None, api_version: str | None, ) -> str | None: + model_info: Final = get_model_info(model=model) if model in litellm.model_cost else {} + supported_endpoints: Final = model_info.get("supported_endpoints") + supported_formats: Final = model_info.get("supported_transcription_response_formats") or ( + _bundled_transcription_response_formats().get(model) + ) if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", model=model, llm_provider=custom_llm_provider, ) - if model == "gpt-live-transcribe": + if supported_endpoints is not None and "/v1/audio/transcriptions" not in supported_endpoints: raise litellm.UnsupportedParamsError( - message="gpt-live-transcribe is available through the Realtime API, not file transcription", + message=f"{model} is available through the Realtime API, not file transcription", model=model, llm_provider=custom_llm_provider, ) - if model == "gpt-transcribe" and response_format not in (None, "json"): + if supported_formats is not None and response_format is not None and response_format not in supported_formats: raise litellm.UnsupportedParamsError( - message="gpt-transcribe only supports response_format='json'", + message=f"{model} only supports response_format={', '.join(repr(fmt) for fmt in supported_formats)}", model=model, llm_provider=custom_llm_provider, ) - if custom_llm_provider == "azure" and model == "gpt-transcribe": + if custom_llm_provider == "azure" and supported_formats is not None: if api_version in (None, "v1", "latest", "preview"): return litellm.AZURE_DEFAULT_API_VERSION return api_version diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 48c83fb891e..b2f8636877c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -214,6 +214,9 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "audio", "text" @@ -63371,6 +63374,9 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "text", "audio" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a9ef1f04a8e..b259c07f63a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -465,6 +465,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): ] ] supported_endpoints: list[str] | None + supported_transcription_response_formats: ReadOnly[Sequence[str] | None] use_openai_responses_path: bool | None tpm: int | None rpm: int | None diff --git a/litellm/utils.py b/litellm/utils.py index 4dfd59540de..cc6ce19e953 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6940,6 +6940,9 @@ def get_model_info_helper( litellm_provider=_model_info.get("litellm_provider", custom_llm_provider), mode=_model_info.get("mode"), supported_endpoints=_model_info.get("supported_endpoints", None), + supported_transcription_response_formats=_model_info.get( + "supported_transcription_response_formats", None + ), supports_system_messages=_model_info.get("supports_system_messages", None), supports_response_schema=_model_info.get("supports_response_schema", None), supports_vision=_model_info.get("supports_vision", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 48c83fb891e..b2f8636877c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -214,6 +214,9 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "audio", "text" @@ -63371,6 +63374,9 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "text", "audio" diff --git a/tests/unit/cookbook/test_gpt_realtime_translate.py b/tests/unit/cookbook/test_gpt_realtime_translate.py new file mode 100644 index 00000000000..e87005d1bad --- /dev/null +++ b/tests/unit/cookbook/test_gpt_realtime_translate.py @@ -0,0 +1,43 @@ +import asyncio +import base64 +import json +import wave +from pathlib import Path +from types import SimpleNamespace +from typing import Final, cast + +import pytest +from websockets.asyncio.client import ClientConnection + +from cookbook import gpt_realtime_translate as translate + + +@pytest.mark.asyncio +async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(translate, "OUTPUT_IDLE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(translate, "INITIAL_RESPONSE_TIMEOUT_SECONDS", 0.1) + audio: Final = bytes(480) + events: Final = iter( + ( + {"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}, + {"type": "error", "error": {"message": "session closed"}}, + ) + ) + + async def recv() -> str: + event: Final = next(events) + if event["type"] == "session.output_audio.delta": + await asyncio.sleep(0.03) + return json.dumps(event) + + sender_finished: Final = asyncio.Event() + sender_finished.set() + output: Final = tmp_path / "translation.wav" + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished + ) + + assert result == 'Realtime API error: {"message": "session closed"}' + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(240) == audio diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 4901f574769..6ddd69b6283 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -1208,7 +1208,7 @@ async def test_translation_session_update_rejects_disallowed_nested_transcriptio translation_session=True, ) - with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available"): await streaming._send_to_backend( json.dumps( { @@ -3043,7 +3043,9 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] -@pytest.mark.parametrize("event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"]) +@pytest.mark.parametrize( + "event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"] +) def test_translation_audio_duration_is_finalized_once(event_type: str): import base64 @@ -3107,6 +3109,29 @@ def test_translation_does_not_duplicate_provider_duration_usage(): assert len(closed_events) == 1 +def test_translation_prefers_provider_duration_over_audio_byte_estimate(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 0.5} + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() From 3fb2252376e53ba36f16af9495145486d03d42d8 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 19:16:22 -0500 Subject: [PATCH 09/45] fix(realtime): keep transcription validation compatible with cost map guard --- litellm/main.py | 28 +++---------------- ...odel_prices_and_context_window_backup.json | 6 ---- litellm/types/utils.py | 1 - litellm/utils.py | 3 -- model_prices_and_context_window.json | 6 ---- 5 files changed, 4 insertions(+), 40 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 53b63de1e9e..1eb46410733 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -23,7 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy -from functools import lru_cache, partial +from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -8248,23 +8248,6 @@ async def atranscription( ) -@lru_cache(maxsize=1) -def _bundled_transcription_response_formats() -> Mapping[str, tuple[str, ...]]: - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - - catalog: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python( - GetModelCostMap.load_local_model_cost_map() - ) - formats_adapter: Final = TypeAdapter(tuple[str, ...]) - return MappingProxyType( - { - model: formats_adapter.validate_python(entry["supported_transcription_response_formats"]) - for model, entry in catalog.items() - if "supported_transcription_response_formats" in entry - } - ) - - def _validate_gpt_transcription_request( model: str, custom_llm_provider: str, @@ -8275,9 +8258,6 @@ def _validate_gpt_transcription_request( ) -> str | None: model_info: Final = get_model_info(model=model) if model in litellm.model_cost else {} supported_endpoints: Final = model_info.get("supported_endpoints") - supported_formats: Final = model_info.get("supported_transcription_response_formats") or ( - _bundled_transcription_response_formats().get(model) - ) if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", @@ -8290,13 +8270,13 @@ def _validate_gpt_transcription_request( model=model, llm_provider=custom_llm_provider, ) - if supported_formats is not None and response_format is not None and response_format not in supported_formats: + if model == "gpt-transcribe" and response_format not in (None, "json"): raise litellm.UnsupportedParamsError( - message=f"{model} only supports response_format={', '.join(repr(fmt) for fmt in supported_formats)}", + message="gpt-transcribe only supports response_format='json'", model=model, llm_provider=custom_llm_provider, ) - if custom_llm_provider == "azure" and supported_formats is not None: + if custom_llm_provider == "azure" and model == "gpt-transcribe": if api_version in (None, "v1", "latest", "preview"): return litellm.AZURE_DEFAULT_API_VERSION return api_version diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b2f8636877c..48c83fb891e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -214,9 +214,6 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "audio", "text" @@ -63374,9 +63371,6 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "text", "audio" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b259c07f63a..a9ef1f04a8e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -465,7 +465,6 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): ] ] supported_endpoints: list[str] | None - supported_transcription_response_formats: ReadOnly[Sequence[str] | None] use_openai_responses_path: bool | None tpm: int | None rpm: int | None diff --git a/litellm/utils.py b/litellm/utils.py index cc6ce19e953..4dfd59540de 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6940,9 +6940,6 @@ def get_model_info_helper( litellm_provider=_model_info.get("litellm_provider", custom_llm_provider), mode=_model_info.get("mode"), supported_endpoints=_model_info.get("supported_endpoints", None), - supported_transcription_response_formats=_model_info.get( - "supported_transcription_response_formats", None - ), supports_system_messages=_model_info.get("supports_system_messages", None), supports_response_schema=_model_info.get("supports_response_schema", None), supports_vision=_model_info.get("supports_vision", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b2f8636877c..48c83fb891e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -214,9 +214,6 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "audio", "text" @@ -63374,9 +63371,6 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "text", "audio" From 366d7f7fd994a24fa852f453506f16137dffa3cb Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 20:16:11 -0500 Subject: [PATCH 10/45] fix(realtime): address transcription and billing review findings --- litellm/__init__.py | 2 - litellm/constants.py | 31 ----- litellm/cost_calculator.py | 34 ++++-- .../audio_utils/transcription_streaming.py | 4 + .../litellm_core_utils/realtime_streaming.py | 5 +- litellm/llms/azure/audio_transcriptions.py | 6 +- litellm/llms/azure/realtime/handler.py | 19 +++- .../azure/realtime/http_transformation.py | 21 +++- .../audio_transcription/transformation.py | 17 +++ .../base_llm/realtime/http_transformation.py | 11 ++ litellm/llms/openai/transcriptions/handler.py | 22 +--- litellm/main.py | 16 ++- ...odel_prices_and_context_window_backup.json | 41 +++++++ litellm/realtime_api/main.py | 52 +++------ litellm/types/llms/openai.py | 1 + model_prices_and_context_window.json | 41 +++++++ .../test_realtime_streaming.py | 4 +- .../realtime/test_transcription_sessions.py | 10 +- .../llms/openai/realtime/test_translation.py | 12 ++ .../transcriptions/test_gpt_transcribe.py | 106 ++++++++++++++++-- .../public_endpoints/test_public_endpoints.py | 29 ++--- tests/unit/realtime_api/test_main.py | 23 ++++ tests/unit/test_cost_calculator.py | 20 ++++ 23 files changed, 383 insertions(+), 144 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e241a2baa1..a1de9f90f5e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -53,7 +53,6 @@ from litellm.constants import ( OPENAI_CHAT_COMPLETION_PARAMS as _openai_completion_params, # backwards compatibility OPENAI_FINISH_REASONS, OPENAI_FINISH_REASONS as _openai_finish_reasons, # backwards compatibility - OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS, openai_compatible_endpoints, openai_compatible_providers, openai_text_completion_compatible_providers, @@ -1035,7 +1034,6 @@ def add_known_models(model_cost_map: Optional[Dict] = None): _populate_provider_model_sets(model_cost) -open_ai_chat_completion_models.update(OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS) # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary # this is maintained for Exception Mapping diff --git a/litellm/constants.py b/litellm/constants.py index bc2ba343aa3..d2cb632b28b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -950,37 +950,6 @@ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "timestamp_granularities", ] -OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS: Final = frozenset( - { - "gpt-realtime-2", - "gpt-realtime-2.1", - "gpt-realtime-2.1-mini", - "gpt-realtime-translate", - "gpt-realtime-whisper", - "gpt-transcribe", - "gpt-live-transcribe", - } -) - -AZURE_GA_REALTIME_MODELS: Final = frozenset( - { - "gpt-realtime-2", - "gpt-realtime-2-2026-05-06", - "gpt-realtime-2.1", - "gpt-realtime-2.1-2026-07-07", - "gpt-realtime-2.1-mini", - "gpt-realtime-2.1-mini-2026-07-07", - "gpt-realtime-translate", - "gpt-realtime-translate-2026-05-06", - "gpt-realtime-translate-2026-05-07", - "gpt-realtime-whisper", - "gpt-realtime-whisper-2026-05-06", - "gpt-realtime-whisper-2026-05-07", - "gpt-transcribe", - "gpt-live-transcribe", - } -) - OPENAI_EMBEDDING_PARAMS: Final = ["dimensions", "encoding_format", "user"] DEFAULT_EMBEDDING_PARAM_VALUES: Final = { diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b65bd02a911..ccb9dd94bf3 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -3647,14 +3647,21 @@ def handle_realtime_translation_cost_calculation( custom_llm_provider: str, litellm_model_name: str, ) -> float: - output_seconds = 0.0 # rebind-ok: duration is accumulated across translation close events - for result in results: - if result.get("type") != _TRANSLATION_CLOSED_EVENT_TYPE: - continue - usage = result.get("usage") - if isinstance(usage, dict) and isinstance(usage.get("output_seconds"), (int, float)): - output_seconds += float(usage["output_seconds"]) - if output_seconds <= 0: + usage_events: Final = tuple( + usage + for result in results + if result.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE + if isinstance(usage := result.get("usage"), dict) + ) + input_seconds: Final = sum( + float(usage["input_seconds"]) for usage in usage_events if isinstance(usage.get("input_seconds"), (int, float)) + ) + output_seconds: Final = sum( + float(usage["output_seconds"]) + for usage in usage_events + if isinstance(usage.get("output_seconds"), (int, float)) + ) + if input_seconds <= 0 and output_seconds <= 0: return 0.0 try: model_info: Final = litellm.get_model_info( @@ -3663,10 +3670,15 @@ def handle_realtime_translation_cost_calculation( ) except Exception: # noqa: BLE001 # unknown model metadata should yield zero translation cost return 0.0 + input_cost_per_second: Final = model_info.get("input_cost_per_second") output_cost_per_second: Final = model_info.get("output_cost_per_second") - if not isinstance(output_cost_per_second, (int, float)): - return 0.0 - return output_seconds * output_cost_per_second + input_cost: Final = ( + input_seconds * input_cost_per_second if isinstance(input_cost_per_second, (int, float)) else 0.0 + ) + output_cost: Final = ( + output_seconds * output_cost_per_second if isinstance(output_cost_per_second, (int, float)) else 0.0 + ) + return input_cost + output_cost def handle_realtime_transcription_cost_calculation( diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 5ae246f3468..979a1847362 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -120,6 +120,8 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): if self._finalized or self._failed: return self._finalized = True + if self._collector.done_event is None and self._collector.duration is None: + return self._logging_obj.success_handler( self._collector.response(), self._start_time, @@ -174,6 +176,8 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): if self._finalized or self._failed: return self._finalized = True + if self._collector.done_event is None and self._collector.duration is None: + return response: Final = self._collector.response() end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract self._logging_obj.handle_sync_success_callbacks_for_async_calls( diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 17824a281ca..6db2ac42f1a 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -565,12 +565,15 @@ class RealTimeStreaming: usage: Final = event_obj.get("usage") output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None if isinstance(output_seconds, (int, float)): + input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None if not self._should_store_message(event_obj): self.messages.append( OpenAIRealtimeTranslationClosedEvent( type="session.closed", usage=OpenAIRealtimeTranslationDurationUsage( - type="duration", output_seconds=output_seconds + type="duration", + output_seconds=output_seconds, + **({"input_seconds": input_seconds} if isinstance(input_seconds, (int, float)) else {}), ), ) ) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index f8e4875ab34..8acf004274e 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -6,6 +6,7 @@ from pydantic import BaseModel from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name +from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data from litellm.types.utils import FileTypes from litellm.utils import ( TranscriptionResponse, @@ -40,11 +41,12 @@ class AzureAudioTranscription(AzureChatCompletion): custom_llm_provider: str = "azure", ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} + sdk_data: Final = sdk_compatible_transcription_request_data(data) if atranscription is True: return self.async_audio_transcriptions( audio_file=audio_file, - data=data, + data=sdk_data, model_response=model_response, timeout=timeout, api_key=api_key, @@ -86,7 +88,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) response: Final = azure_client.audio.transcriptions.create( - **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 4e93047b8a0..191857bff57 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -8,6 +8,7 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final, Protocol, cast +import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.url_utils import join_api_base @@ -48,16 +49,30 @@ async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTe pass +def azure_realtime_requires_ga(model: str) -> bool: + try: + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + return False + return (model_info.get("provider_specific_entry") or {}).get("realtime_ga_only") == 1 + + def azure_realtime_protocol_for_client( configured_protocol: object, *, + model: str, + realtime_mode: str, query_params: RealtimeQueryParams | None, websocket: ScopedWebSocket, ) -> str: + if azure_realtime_requires_ga(model): + if isinstance(configured_protocol, str) and configured_protocol.upper() not in ("GA", "V1"): + raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") + return "GA" + if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": + return "GA" if isinstance(configured_protocol, str) and configured_protocol: return configured_protocol - if (query_params or {}).get("intent") == "transcription": - return "GA" return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index ea2c2990182..285d7f487ff 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -1,12 +1,16 @@ """Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" +from collections.abc import Mapping from typing import Final import litellm -from litellm.constants import AZURE_GA_REALTIME_MODELS from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.azure.common_utils import get_azure_ad_token from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +from .handler import azure_realtime_requires_ga def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str: @@ -18,7 +22,7 @@ def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str: class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): @staticmethod def _uses_ga_api(model: str, api_version: str | None) -> bool: - return api_version in ("preview", "latest", "v1") or model in AZURE_GA_REALTIME_MODELS + return api_version in ("preview", "latest", "v1") or azure_realtime_requires_ga(model) def get_api_base(self, api_base: str | None, **kwargs: object) -> str: return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or "" @@ -26,6 +30,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): def get_api_key(self, api_key: str | None, **kwargs: object) -> str: return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or "" + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + if api_key: + return extra_headers + azure_ad_token: Final = get_azure_ad_token(litellm_params) + if azure_ad_token is None: + return extra_headers + return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"} + def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets") diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..2078b063453 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -11,6 +11,23 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse + +def sdk_compatible_transcription_request_data(data: dict) -> dict: + extension_keys: Final = ("keywords", "languages") + extension_body: Final = {key: data[key] for key in extension_keys if key in data} + if not extension_body: + return data + + existing_extra_body: Final = data.get("extra_body") + return { + **{key: value for key, value in data.items() if key not in extension_keys}, + "extra_body": { + **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), + **extension_body, + }, + } + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index 18363d288e4..c1c943c2cd6 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow: """ from abc import ABC, abstractmethod +from collections.abc import Mapping +from typing import Final import httpx from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.types.router import GenericLiteLLMParams class BaseRealtimeHTTPConfig(ABC): @@ -50,6 +53,14 @@ class BaseRealtimeHTTPConfig(ABC): explicit api_key → litellm.api_key → env var → "" """ + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + return extra_headers + # ------------------------------------------------------------------ # # client_secrets endpoint # # ------------------------------------------------------------------ # diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index eb523f3f050..f2d46a50ddd 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -12,6 +12,7 @@ from litellm.litellm_core_utils.core_helpers import set_provider_response_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, + sdk_compatible_transcription_request_data, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -25,23 +26,6 @@ from ..openai import OpenAIChatCompletion class OpenAIAudioTranscription(OpenAIChatCompletion): # Audio Transcriptions - @staticmethod - def _sdk_compatible_request_data(data: dict) -> dict: - """Route API fields that predate SDK support through ``extra_body``.""" - extension_keys: Final = ("keywords", "languages") - extension_body: Final = {key: data[key] for key in extension_keys if key in data} - if not extension_body: - return data - - existing_extra_body: Final = data.get("extra_body") - return { # mutable-ok: OpenAI SDK requires a mutable request mapping - **{key: value for key, value in data.items() if key not in extension_keys}, - "extra_body": { - **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), - **extension_body, - }, - } - async def make_openai_audio_transcriptions_request( self, openai_aclient: AsyncOpenAI, @@ -49,7 +33,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout: float | httpx.Timeout, ): try: - sdk_data: Final = self._sdk_compatible_request_data(data) + sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout) return {}, stream_response # mutable-ok: response headers use the existing mutable mapping contract @@ -70,7 +54,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout: float | httpx.Timeout, ): try: - sdk_data: Final = self._sdk_compatible_request_data(data) + sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: response = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) return None, response diff --git a/litellm/main.py b/litellm/main.py index 1eb46410733..ae9100c8bba 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8256,8 +8256,14 @@ def _validate_gpt_transcription_request( response_format: str | None, api_version: str | None, ) -> str | None: - model_info: Final = get_model_info(model=model) if model in litellm.model_cost else {} + model_cost_key: Final = f"{custom_llm_provider}/{model}" if custom_llm_provider == "azure" else model + model_info: Final = ( + get_model_info(model=model, custom_llm_provider=custom_llm_provider) + if model_cost_key in litellm.model_cost + else {} + ) supported_endpoints: Final = model_info.get("supported_endpoints") + provider_specific_entry: Final = model_info.get("provider_specific_entry") or {} if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", @@ -8270,14 +8276,14 @@ def _validate_gpt_transcription_request( model=model, llm_provider=custom_llm_provider, ) - if model == "gpt-transcribe" and response_format not in (None, "json"): + if provider_specific_entry.get("transcription_json_only") == 1 and response_format not in (None, "json"): raise litellm.UnsupportedParamsError( - message="gpt-transcribe only supports response_format='json'", + message=f"{model} only supports response_format='json'", model=model, llm_provider=custom_llm_provider, ) - if custom_llm_provider == "azure" and model == "gpt-transcribe": - if api_version in (None, "v1", "latest", "preview"): + if provider_specific_entry.get("transcription_deployment_api") == 1: + if api_version in ("v1", "latest", "preview"): return litellm.AZURE_DEFAULT_API_VERSION return api_version return api_version diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 48c83fb891e..8b6b9e28d92 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3,6 +3,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/realtime", @@ -30,6 +33,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -67,6 +73,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -98,6 +107,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -122,6 +134,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -146,6 +161,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", "supported_endpoints": [ @@ -168,6 +186,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -188,6 +209,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -208,6 +232,11 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -6772,6 +6801,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6810,6 +6842,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -7018,6 +7053,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -63366,6 +63404,9 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_only": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index d658cf787d2..969d49b37dc 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -13,7 +13,6 @@ import httpx import litellm from litellm.constants import ( - AZURE_GA_REALTIME_MODELS, AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, @@ -58,8 +57,6 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket - from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig - azure_realtime: Final = AzureOpenAIRealtime() openai_realtime: Final = OpenAIRealtime() openai_live_sessions: Final = OpenAILiveSessions() @@ -120,21 +117,6 @@ def _build_litellm_metadata(kwargs: dict) -> dict[str, object]: return metadata -def _resolve_azure_realtime_protocol( - model: str, - realtime_protocol: str | None, - query_params: RealtimeQueryParams | None, - realtime_mode: str, -) -> str: - if model in AZURE_GA_REALTIME_MODELS: - if realtime_protocol is not None and realtime_protocol.upper() not in ("GA", "V1"): - raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") - return "GA" - if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": - return "GA" - return realtime_protocol or "beta" - - def _get_realtime_http_provider_config( custom_llm_provider: str, dynamic_api_base: str | None, @@ -183,19 +165,18 @@ async def _realtime_http_bearer_token( def _get_realtime_http_extra_headers( - custom_llm_provider: str, + provider_config: BaseRealtimeHTTPConfig | None, litellm_params: GenericLiteLLMParams, resolved_api_key: str, extra_headers: Mapping[str, object] | None, ) -> Mapping[str, object] | None: - resolved_headers: Final = { # mutable-ok: Azure authentication may extend caller-supplied headers - **(extra_headers or {}) - } - if custom_llm_provider == "azure" and not resolved_api_key: - azure_ad_token: Final = get_azure_ad_token(litellm_params) - if azure_ad_token: - resolved_headers["Authorization"] = f"Bearer {azure_ad_token}" - return resolved_headers or None + if provider_config is None: + return extra_headers + return provider_config.get_extra_headers( + litellm_params=litellm_params, + api_key=resolved_api_key, + extra_headers=extra_headers, + ) @wrapper_client @@ -249,7 +230,7 @@ async def acreate_realtime_client_secret( litellm_params=litellm_params, ) resolved_extra_headers: Final = _get_realtime_http_extra_headers( - custom_llm_provider=custom_llm_provider, + provider_config=provider_config, litellm_params=litellm_params, resolved_api_key=resolved_api_key, extra_headers=kwargs.get("extra_headers"), @@ -324,7 +305,7 @@ async def acreate_realtime_translation_client_secret( litellm_params=litellm_params, ) resolved_extra_headers: Final = _get_realtime_http_extra_headers( - custom_llm_provider=custom_llm_provider, + provider_config=provider_config, litellm_params=litellm_params, resolved_api_key=resolved_api_key, extra_headers=kwargs.get("extra_headers"), @@ -408,7 +389,7 @@ async def acreate_realtime_transcription_session( litellm_params=litellm_params, ) resolved_extra_headers: Final = _get_realtime_http_extra_headers( - custom_llm_provider=custom_llm_provider, + provider_config=provider_config, litellm_params=litellm_params, resolved_api_key=resolved_api_key, extra_headers=kwargs.get("extra_headers"), @@ -793,17 +774,12 @@ async def _arealtime( GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), ) ) - realtime_protocol: Final = _resolve_azure_realtime_protocol( + realtime_protocol: Final = azure_realtime_protocol_for_client( + configured_realtime_protocol, model=model, - realtime_protocol=( - configured_realtime_protocol - if model in AZURE_GA_REALTIME_MODELS or realtime_mode == "translation" - else azure_realtime_protocol_for_client( - configured_realtime_protocol, query_params=query_params, websocket=websocket - ) - ), query_params=query_params, realtime_mode=realtime_mode, + websocket=websocket, ) await azure_realtime.async_realtime( model=model, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 4c0e6a78d4b..0e32f2d5e0d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2444,6 +2444,7 @@ class OpenAIRealtimeResponseUsage(TypedDict): class OpenAIRealtimeTranslationDurationUsage(TypedDict): type: ReadOnly[Literal["duration"]] output_seconds: ReadOnly[float] + input_seconds: NotRequired[ReadOnly[float]] class OpenAIRealtimeTranslationClosedEvent(TypedDict): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 48c83fb891e..8b6b9e28d92 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3,6 +3,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/realtime", @@ -30,6 +33,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -67,6 +73,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -98,6 +107,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -122,6 +134,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -146,6 +161,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", "supported_endpoints": [ @@ -168,6 +186,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -188,6 +209,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -208,6 +232,11 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -6772,6 +6801,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6810,6 +6842,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -7018,6 +7053,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -63366,6 +63404,9 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_only": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 6ddd69b6283..c89f02e90d4 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3123,13 +3123,13 @@ def test_translation_prefers_provider_duration_over_audio_byte_estimate(): {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} ) streaming._capture_translation_output_audio( - {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}} + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5}} ) streaming._finalize_translation_usage() closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] assert len(closed_events) == 1 - assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 0.5} + assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5} @pytest.mark.asyncio diff --git a/tests/unit/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py index 27ffcf26878..5b6945282dc 100644 --- a/tests/unit/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/unit/llms/openai/realtime/test_transcription_sessions.py @@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import litellm from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -46,15 +47,18 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version(): @pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) -def test_azure_ga_realtime_http_urls(api_version): +def test_azure_ga_realtime_http_urls(api_version, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() cfg = AzureRealtimeHTTPConfig() base = "https://my.openai.azure.com" - assert cfg.get_complete_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/client_secrets") - assert cfg.get_realtime_calls_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/calls") + assert cfg.get_complete_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/calls") assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == ( f"{base}/openai/v1/realtime/transcription_sessions" ) + litellm.get_model_info.cache_clear() def test_request_resolves_model_returns_none_when_both_absent(): diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py index a329200b2f7..06af49f26ca 100644 --- a/tests/unit/llms/openai/realtime/test_translation.py +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -11,6 +11,18 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig from litellm.types.realtime import RealtimeSessionConfig +from litellm.types.router import GenericLiteLLMParams + + +def test_azure_realtime_http_headers_use_ad_token_without_api_key(): + config = AzureRealtimeHTTPConfig() + params = GenericLiteLLMParams(azure_ad_token="test-ad-token") + + assert config.get_extra_headers(params, "", {"X-Trace": "trace"}) == { + "X-Trace": "trace", + "Authorization": "Bearer test-ad-token", + } + assert config.get_extra_headers(params, "test-api-key", {"X-Trace": "trace"}) == {"X-Trace": "trace"} def test_realtime_session_config_supports_translation_and_live_transcription_fields(): diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 35bed209400..742e1852ea5 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -1,12 +1,13 @@ import io import json import wave +from collections.abc import Iterator from datetime import datetime from unittest.mock import AsyncMock, MagicMock import httpx import pytest -from openai import AsyncOpenAI, AsyncStream, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI import litellm from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream @@ -20,6 +21,14 @@ from litellm.types.utils import TranscriptionResponse from litellm.utils import get_optional_params_transcription +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + def test_gpt_transcribe_config_uses_native_parameters_and_json(): config = OpenAIGPTTranscribeAudioTranscriptionConfig() supported = config.get_supported_openai_params("gpt-transcribe") @@ -122,6 +131,37 @@ async def test_openai_handler_returns_native_typed_stream(): assert logged_response.languages == [] +@pytest.mark.asyncio +async def test_closed_transcription_stream_without_usage_or_duration_does_not_log_success(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello"}, + {"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 1}}, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + stream = await client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now()) + + async for event in wrapped_stream: + assert event.type == "transcript.text.delta" + break + await wrapped_stream.close() + await client.close() + + logging_obj.handle_sync_success_callbacks_for_async_calls.assert_not_called() + logging_obj.async_success_handler.assert_not_awaited() + logging_obj.async_failure_handler.assert_not_awaited() + + @pytest.mark.asyncio async def test_atranscription_stream_preserves_duration_for_callback_cost(): async def send_response(request: httpx.Request) -> httpx.Response: @@ -188,19 +228,20 @@ def test_gpt_transcribe_rejects_conflicting_language_inputs(): ) -def test_gpt_transcribe_rejects_whisper_response_formats(): +@pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"]) +def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None: audio_file = io.BytesIO(b"audio") audio_file.name = "sample.wav" with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format='json'"): litellm.transcription( - model="gpt-transcribe", + model=model, file=audio_file, response_format="verbose_json", api_key="sk-test", ) -def test_gpt_live_transcribe_rejects_file_transcription(): +def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: None) -> None: audio_file = io.BytesIO(b"audio") audio_file.name = "sample.wav" with pytest.raises(litellm.UnsupportedParamsError, match="Realtime API"): @@ -232,8 +273,8 @@ def test_azure_async_gpt_transcribe_forwards_v1_api_version(): assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == "v1" -@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) -def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: str | None): +@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) +def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(local_model_cost_map: None, api_version: str) -> None: resolved_api_version = _validate_gpt_transcription_request( model="gpt-transcribe", custom_llm_provider="azure", @@ -246,6 +287,20 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: st assert resolved_api_version == litellm.AZURE_DEFAULT_API_VERSION +def test_azure_gpt_transcribe_keeps_unset_api_version_for_configured_default(local_model_cost_map: None) -> None: + assert ( + _validate_gpt_transcription_request( + model="gpt-transcribe", + custom_llm_provider="azure", + language=None, + languages=None, + response_format="json", + api_version=None, + ) + is None + ) + + def test_azure_gpt_transcribe_uses_deployment_scoped_route(): def send_response(request: httpx.Request) -> httpx.Response: assert str(request.url) == ( @@ -287,7 +342,7 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_route(): client.close() -def test_azure_gpt_transcribe_preserves_dated_api_version(): +def test_azure_gpt_transcribe_preserves_dated_api_version(local_model_cost_map: None) -> None: resolved_api_version = _validate_gpt_transcription_request( model="gpt-transcribe", custom_llm_provider="azure", @@ -298,3 +353,40 @@ def test_azure_gpt_transcribe_preserves_dated_api_version(): ) assert resolved_api_version == "2025-04-01-preview" + + +@pytest.mark.asyncio +async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + return httpx.Response(200, json={"text": "hello"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncAzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version="2025-04-01-preview", + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = await AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"]}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version="2025-04-01-preview", + client=client, + atranscription=True, + ) + + assert response.text == "hello" + await client.close() diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 8f27365fcc1..0107b89fe99 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -102,29 +102,20 @@ def test_get_litellm_model_cost_map_catalog_only_excludes_runtime_registered_ent ) -def test_get_litellm_model_cost_map_returns_cost_map(): - app = FastAPI() +def test_get_litellm_model_cost_map_returns_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + model_cost: Final = { + "audio-model": {"input_cost_per_second": 0.1, "mode": "audio_transcription"}, + "token-model": {"input_cost_per_token": 0.01, "mode": "chat"}, + } + monkeypatch.setattr(litellm, "model_cost", model_cost) + app: Final = FastAPI() app.include_router(router) - client = TestClient(app) + client: Final = TestClient(app) - response = client.get("/public/litellm_model_cost_map") + response: Final = client.get("/public/litellm_model_cost_map") assert response.status_code == 200 - payload = response.json() - assert isinstance(payload, dict) - assert len(payload) > 0, "Expected model cost map to contain at least one model" - - # Verify the structure contains expected keys for at least one model - # Check for a common model like gpt-4 or gpt-3.5-turbo - model_keys = list(payload.keys()) - assert len(model_keys) > 0 - - # Verify at least one model has expected cost fields - sample_model = model_keys[0] - sample_model_data = payload[sample_model] - assert isinstance(sample_model_data, dict) - # Check for common cost fields that should be present - assert "input_cost_per_token" in sample_model_data or "output_cost_per_token" in sample_model_data + assert response.json() == model_cost def test_public_ai_hub_info_is_public_by_default(monkeypatch): diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 4d89ee497a5..bf2f5733463 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -743,6 +743,29 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) +def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + assert ( + azure_realtime_protocol_for_client( + None, + model="gpt-realtime-2.1", + realtime_mode="realtime", + query_params=None, + websocket=_BETA_CLIENT, + ) + == "GA" + ) + with pytest.raises(ValueError, match="requires the Azure OpenAI v1 Realtime API"): + azure_realtime_protocol_for_client( + "beta", + model="gpt-realtime-2.1", + realtime_mode="realtime", + query_params=None, + websocket=_BETA_CLIENT, + ) + + async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: connect: Final = _ConnectThatStopsAfterCapturingTheUrl() with patch("websockets.connect", connect): diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index ef72c61df7e..a8b5377caf2 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10458,3 +10458,23 @@ def test_realtime_translation_duration_cost(_local_model_cost_map): ) assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) + + +def test_realtime_translation_duration_cost_includes_provider_input_usage(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 3.0, "output_seconds": 2.0}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + expected: Final = ( + 3 * litellm.model_cost[model]["input_cost_per_second"] + + 2 * litellm.model_cost[model]["output_cost_per_second"] + ) + assert actual == pytest.approx(expected) From 27115706ceeb8370e1ca76ce41299e5c59cb059a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 03:09:01 -0500 Subject: [PATCH 11/45] fix(realtime): preserve input usage and provider routing --- .../litellm_core_utils/realtime_streaming.py | 37 ++++++++---- litellm/llms/azure/audio_transcriptions.py | 18 +++++- litellm/llms/azure/realtime/handler.py | 12 +++- litellm/main.py | 33 +++++----- ...odel_prices_and_context_window_backup.json | 3 + litellm/types/llms/openai.py | 2 +- model_prices_and_context_window.json | 3 + .../test_realtime_streaming.py | 28 +++++++++ .../realtime/test_transcription_sessions.py | 7 ++- .../transcriptions/test_gpt_transcribe.py | 60 +++++-------------- tests/unit/realtime_api/test_main.py | 7 ++- tests/unit/test_cost_calculator.py | 16 ++++- 12 files changed, 137 insertions(+), 89 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 6db2ac42f1a..a009fe61b61 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -564,18 +564,30 @@ class RealTimeStreaming: if event_obj.get("type") == "session.closed": usage: Final = event_obj.get("usage") output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None - if isinstance(output_seconds, (int, float)): - input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None + input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None + synthetic_output_seconds: Final = ( + self._translation_output_audio_bytes / self._translation_output_bytes_per_second + if self._translation_output_audio_bytes > 0 + else None + ) + resolved_output_seconds: Final = ( + output_seconds if isinstance(output_seconds, (int, float)) else synthetic_output_seconds + ) + if isinstance(input_seconds, (int, float)) or resolved_output_seconds is not None: if not self._should_store_message(event_obj): - self.messages.append( - OpenAIRealtimeTranslationClosedEvent( - type="session.closed", - usage=OpenAIRealtimeTranslationDurationUsage( - type="duration", - output_seconds=output_seconds, - **({"input_seconds": input_seconds} if isinstance(input_seconds, (int, float)) else {}), - ), + normalized_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=float(input_seconds), + output_seconds=float(resolved_output_seconds or 0.0), ) + if isinstance(input_seconds, (int, float)) + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=float(resolved_output_seconds or 0.0) + ) + ) + self.messages.append( + OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=normalized_usage) ) self._translation_usage_finalized = True return @@ -624,7 +636,10 @@ class RealTimeStreaming: if event.get("type") != "session.closed": continue event_usage = event.get("usage") # rebind-ok: each close event carries independent usage - if isinstance(event_usage, dict) and isinstance(event_usage.get("output_seconds"), (int, float)): + if isinstance(event_usage, dict) and ( + isinstance(event_usage.get("input_seconds"), (int, float)) + or isinstance(event_usage.get("output_seconds"), (int, float)) + ): self._translation_usage_finalized = True return if self._translation_output_audio_bytes == 0: diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 8acf004274e..8f59cd62cb0 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from pydantic import BaseModel +import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data @@ -42,6 +43,19 @@ class AzureAudioTranscription(AzureChatCompletion): ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) + model_info: Final = ( + litellm.get_model_info(model=model, custom_llm_provider="azure") + if f"azure/{model}" in litellm.model_cost + else None + ) + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + resolved_api_version: Final = ( + litellm.AZURE_DEFAULT_API_VERSION + if provider_specific_entry is not None + and provider_specific_entry.get("transcription_deployment_api") == 1 + and api_version in ("v1", "latest", "preview") + else api_version + ) if atranscription is True: return self.async_audio_transcriptions( @@ -51,7 +65,7 @@ class AzureAudioTranscription(AzureChatCompletion): timeout=timeout, api_key=api_key, api_base=api_base, - api_version=api_version, + api_version=resolved_api_version, client=client, max_retries=max_retries, logging_obj=logging_obj, @@ -61,7 +75,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) azure_client: Final = self.get_azure_openai_client( - api_version=api_version, + api_version=resolved_api_version, api_base=api_base, api_key=api_key, model=model, diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 191857bff57..eb4e02f863b 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -51,10 +51,16 @@ async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTe def azure_realtime_requires_ga(model: str) -> bool: try: - model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") + azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly - return False - return (model_info.get("provider_specific_entry") or {}).get("realtime_ga_only") == 1 + try: + openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + return False + openai_entry: Final = openai_model_info.get("provider_specific_entry") + return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1 + azure_entry: Final = azure_model_info.get("provider_specific_entry") + return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1 def azure_realtime_protocol_for_client( diff --git a/litellm/main.py b/litellm/main.py index ae9100c8bba..1f12b933bee 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8254,16 +8254,15 @@ def _validate_gpt_transcription_request( language: str | None, languages: Sequence[str] | None, response_format: str | None, - api_version: str | None, -) -> str | None: - model_cost_key: Final = f"{custom_llm_provider}/{model}" if custom_llm_provider == "azure" else model - model_info: Final = ( - get_model_info(model=model, custom_llm_provider=custom_llm_provider) - if model_cost_key in litellm.model_cost - else {} +) -> None: + model_cost_key: Final = next( + (key for key in (f"{custom_llm_provider}/{model}", model) if key in litellm.model_cost), None ) - supported_endpoints: Final = model_info.get("supported_endpoints") - provider_specific_entry: Final = model_info.get("provider_specific_entry") or {} + model_info: Final = ( + get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_cost_key is not None else None + ) + supported_endpoints: Final = model_info.get("supported_endpoints") if model_info is not None else None + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", @@ -8276,17 +8275,16 @@ def _validate_gpt_transcription_request( model=model, llm_provider=custom_llm_provider, ) - if provider_specific_entry.get("transcription_json_only") == 1 and response_format not in (None, "json"): + if ( + provider_specific_entry is not None + and provider_specific_entry.get("transcription_json_only") == 1 + and response_format not in (None, "json") + ): raise litellm.UnsupportedParamsError( message=f"{model} only supports response_format='json'", model=model, llm_provider=custom_llm_provider, ) - if provider_specific_entry.get("transcription_deployment_api") == 1: - if api_version in ("v1", "latest", "preview"): - return litellm.AZURE_DEFAULT_API_VERSION - return api_version - return api_version @client @@ -8354,13 +8352,12 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key - validated_api_version: Final = _validate_gpt_transcription_request( + _validate_gpt_transcription_request( model=model, custom_llm_provider=custom_llm_provider, language=language, languages=languages, response_format=response_format, - api_version=api_version, ) optional_params: Final = get_optional_params_transcription( @@ -8416,7 +8413,7 @@ def transcription( # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - azure_api_version: Final = validated_api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + azure_api_version: Final = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8b6b9e28d92..ab086890922 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -38995,6 +38995,9 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0e32f2d5e0d..f333e39087c 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2443,7 +2443,7 @@ class OpenAIRealtimeResponseUsage(TypedDict): class OpenAIRealtimeTranslationDurationUsage(TypedDict): type: ReadOnly[Literal["duration"]] - output_seconds: ReadOnly[float] + output_seconds: NotRequired[ReadOnly[float]] input_seconds: NotRequired[ReadOnly[float]] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8b6b9e28d92..ab086890922 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -38995,6 +38995,9 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index c89f02e90d4..62f33a786b7 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3132,6 +3132,34 @@ def test_translation_prefers_provider_duration_over_audio_byte_estimate(): assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5} +@pytest.mark.parametrize( + ("output_audio_bytes", "expected_usage"), + [ + (0, {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.0}), + (48000, {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0}), + ], +) +def test_translation_preserves_input_only_provider_usage( + output_audio_bytes: int, expected_usage: Mapping[str, str | float] +) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = output_audio_bytes + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == expected_usage + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/unit/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py index 5b6945282dc..514965cd581 100644 --- a/tests/unit/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/unit/llms/openai/realtime/test_transcription_sessions.py @@ -47,14 +47,15 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version(): @pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) -def test_azure_ga_realtime_http_urls(api_version, monkeypatch: pytest.MonkeyPatch): +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_realtime_http_urls(api_version, model: str, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) litellm.get_model_info.cache_clear() cfg = AzureRealtimeHTTPConfig() base = "https://my.openai.azure.com" - assert cfg.get_complete_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/client_secrets") - assert cfg.get_realtime_calls_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/calls") + assert cfg.get_complete_url(base, model, api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, model, api_version) == (f"{base}/openai/v1/realtime/calls") assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == ( f"{base}/openai/v1/realtime/transcription_sessions" ) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 742e1852ea5..26eb6654d87 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -16,7 +16,6 @@ from litellm.llms.openai.transcriptions.gpt_transformation import ( OpenAIGPTTranscribeAudioTranscriptionConfig, ) from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription -from litellm.main import _validate_gpt_transcription_request from litellm.types.utils import TranscriptionResponse from litellm.utils import get_optional_params_transcription @@ -252,7 +251,19 @@ def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: No ) -def test_azure_async_gpt_transcribe_forwards_v1_api_version(): +@pytest.mark.parametrize( + ("api_version", "expected_api_version"), + [ + ("v1", litellm.AZURE_DEFAULT_API_VERSION), + ("latest", litellm.AZURE_DEFAULT_API_VERSION), + ("preview", litellm.AZURE_DEFAULT_API_VERSION), + (None, None), + ("2025-04-01-preview", "2025-04-01-preview"), + ], +) +def test_azure_gpt_transcribe_resolves_api_version_in_provider( + local_model_cost_map: None, api_version: str | None, expected_api_version: str | None +) -> None: handler = AzureAudioTranscription() handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) @@ -266,39 +277,11 @@ def test_azure_async_gpt_transcribe_forwards_v1_api_version(): max_retries=0, api_key="sk-test", api_base="https://example.openai.azure.com", - api_version="v1", + api_version=api_version, atranscription=True, ) - assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == "v1" - - -@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) -def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(local_model_cost_map: None, api_version: str) -> None: - resolved_api_version = _validate_gpt_transcription_request( - model="gpt-transcribe", - custom_llm_provider="azure", - language=None, - languages=None, - response_format="json", - api_version=api_version, - ) - - assert resolved_api_version == litellm.AZURE_DEFAULT_API_VERSION - - -def test_azure_gpt_transcribe_keeps_unset_api_version_for_configured_default(local_model_cost_map: None) -> None: - assert ( - _validate_gpt_transcription_request( - model="gpt-transcribe", - custom_llm_provider="azure", - language=None, - languages=None, - response_format="json", - api_version=None, - ) - is None - ) + assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == expected_api_version def test_azure_gpt_transcribe_uses_deployment_scoped_route(): @@ -342,19 +325,6 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_route(): client.close() -def test_azure_gpt_transcribe_preserves_dated_api_version(local_model_cost_map: None) -> None: - resolved_api_version = _validate_gpt_transcription_request( - model="gpt-transcribe", - custom_llm_provider="azure", - language=None, - languages=None, - response_format="json", - api_version="2025-04-01-preview", - ) - - assert resolved_api_version == "2025-04-01-preview" - - @pytest.mark.asyncio async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): async def send_response(request: httpx.Request) -> httpx.Response: diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index bf2f5733463..b6aca93cd67 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -743,13 +743,14 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) -def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map) -> None: +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map, model: str) -> None: from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client assert ( azure_realtime_protocol_for_client( None, - model="gpt-realtime-2.1", + model=model, realtime_mode="realtime", query_params=None, websocket=_BETA_CLIENT, @@ -759,7 +760,7 @@ def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map) with pytest.raises(ValueError, match="requires the Azure OpenAI v1 Realtime API"): azure_realtime_protocol_for_client( "beta", - model="gpt-realtime-2.1", + model=model, realtime_mode="realtime", query_params=None, websocket=_BETA_CLIENT, diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index a8b5377caf2..f122a71d51a 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10460,12 +10460,22 @@ def test_realtime_translation_duration_cost(_local_model_cost_map): assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) -def test_realtime_translation_duration_cost_includes_provider_input_usage(_local_model_cost_map): +@pytest.mark.parametrize("output_seconds", [None, 2.0]) +def test_realtime_translation_duration_cost_includes_provider_input_usage( + _local_model_cost_map, output_seconds: float | None +): from litellm.cost_calculator import handle_realtime_translation_cost_calculation model: Final = "gpt-realtime-translate" events: Final[OpenAIRealtimeStreamList] = [ - {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 3.0, "output_seconds": 2.0}} + { + "type": "session.closed", + "usage": { + "type": "duration", + "input_seconds": 3.0, + **({"output_seconds": output_seconds} if output_seconds is not None else {}), + }, + } ] actual: Final = handle_realtime_translation_cost_calculation( results=events, @@ -10475,6 +10485,6 @@ def test_realtime_translation_duration_cost_includes_provider_input_usage(_local expected: Final = ( 3 * litellm.model_cost[model]["input_cost_per_second"] - + 2 * litellm.model_cost[model]["output_cost_per_second"] + + (output_seconds or 0) * litellm.model_cost[model]["output_cost_per_second"] ) assert actual == pytest.approx(expected) From 7d79d642f8cb429016c77a3d1be367ed79d3be43 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 03:23:57 -0500 Subject: [PATCH 12/45] fix(realtime): bill retained output and route Azure aliases --- .../litellm_core_utils/realtime_streaming.py | 12 +++++++++- litellm/llms/azure/audio_transcriptions.py | 12 +--------- ...odel_prices_and_context_window_backup.json | 1 - model_prices_and_context_window.json | 1 - .../test_realtime_streaming.py | 22 +++++++++++++++++++ .../transcriptions/test_gpt_transcribe.py | 7 +++--- 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a009fe61b61..1dbe37b0780 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -574,7 +574,17 @@ class RealTimeStreaming: output_seconds if isinstance(output_seconds, (int, float)) else synthetic_output_seconds ) if isinstance(input_seconds, (int, float)) or resolved_output_seconds is not None: - if not self._should_store_message(event_obj): + if self._should_store_message(event_obj): + if not isinstance(output_seconds, (int, float)) and synthetic_output_seconds is not None: + self.messages.append( + OpenAIRealtimeTranslationClosedEvent( + type="session.closed", + usage=OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=synthetic_output_seconds + ), + ) + ) + else: normalized_usage: Final = ( OpenAIRealtimeTranslationDurationUsage( type="duration", diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 8f59cd62cb0..caba5d48314 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -43,18 +43,8 @@ class AzureAudioTranscription(AzureChatCompletion): ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) - model_info: Final = ( - litellm.get_model_info(model=model, custom_llm_provider="azure") - if f"azure/{model}" in litellm.model_cost - else None - ) - provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None resolved_api_version: Final = ( - litellm.AZURE_DEFAULT_API_VERSION - if provider_specific_entry is not None - and provider_specific_entry.get("transcription_deployment_api") == 1 - and api_version in ("v1", "latest", "preview") - else api_version + litellm.AZURE_DEFAULT_API_VERSION if api_version in ("v1", "latest", "preview") else api_version ) if atranscription is True: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ab086890922..6e3da13e476 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -234,7 +234,6 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, - "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ab086890922..6e3da13e476 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -234,7 +234,6 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, - "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 62f33a786b7..db36ebc591c 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3160,6 +3160,28 @@ def test_translation_preserves_input_only_provider_usage( assert closed_events[0]["usage"] == expected_usage +def test_translation_retained_input_only_close_event_bills_captured_output(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*") + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + close_event: Final = {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage_events: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert len(usage_events) == 2 + assert sum(usage.get("input_seconds", 0.0) for usage in usage_events) == 0.25 + assert sum(usage.get("output_seconds", 0.0) for usage in usage_events) == 1.0 + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 26eb6654d87..d192550943c 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -261,14 +261,15 @@ def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: No ("2025-04-01-preview", "2025-04-01-preview"), ], ) -def test_azure_gpt_transcribe_resolves_api_version_in_provider( - local_model_cost_map: None, api_version: str | None, expected_api_version: str | None +@pytest.mark.parametrize("model", ["gpt-transcribe", "custom-transcribe-deployment"]) +def test_azure_audio_transcription_resolves_api_version_in_provider( + model: str, api_version: str | None, expected_api_version: str | None ) -> None: handler = AzureAudioTranscription() handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) handler.audio_transcriptions( - model="gpt-transcribe", + model=model, audio_file=io.BytesIO(b"audio"), optional_params={"stream": True}, logging_obj=MagicMock(), From 2dab37ae536c6d3173464240d88fd43cffd2ff76 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 03:40:09 -0500 Subject: [PATCH 13/45] fix(realtime): reject malformed provider durations --- litellm/cost_calculator.py | 9 ++++-- .../litellm_core_utils/audio_utils/utils.py | 11 +++++++ .../litellm_core_utils/realtime_streaming.py | 27 +++++++++------- litellm/realtime_api/README.md | 14 +-------- .../test_realtime_streaming.py | 21 +++++++++++++ tests/unit/test_cost_calculator.py | 31 +++++++++++++++++++ 6 files changed, 85 insertions(+), 28 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ccb9dd94bf3..c236032dc9f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -3654,12 +3655,14 @@ def handle_realtime_translation_cost_calculation( if isinstance(usage := result.get("usage"), dict) ) input_seconds: Final = sum( - float(usage["input_seconds"]) for usage in usage_events if isinstance(usage.get("input_seconds"), (int, float)) + seconds + for usage in usage_events + if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None ) output_seconds: Final = sum( - float(usage["output_seconds"]) + seconds for usage in usage_events - if isinstance(usage.get("output_seconds"), (int, float)) + if (seconds := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None ) if input_seconds <= 0 and output_seconds <= 0: return 0.0 diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index dab3e48f91a..9fdfce057d9 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -3,6 +3,7 @@ Utils used for litellm.transcription() and litellm.atranscription() """ import hashlib +import math import os from dataclasses import dataclass from typing import Final @@ -17,6 +18,16 @@ from litellm.types.files import ( from litellm.types.utils import FileTypes +def normalized_audio_duration_seconds(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + seconds: Final = float(value) + except OverflowError: + return None + return seconds if math.isfinite(seconds) and seconds >= 0 else None + + @dataclass class ProcessedAudioFile: """ diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 1dbe37b0780..8d05ea17da0 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -20,6 +20,7 @@ from litellm.constants import ( REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( @@ -563,19 +564,21 @@ class RealTimeStreaming: return if event_obj.get("type") == "session.closed": usage: Final = event_obj.get("usage") - output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None - input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None + output_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("output_seconds")) if isinstance(usage, dict) else None + ) + input_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("input_seconds")) if isinstance(usage, dict) else None + ) synthetic_output_seconds: Final = ( self._translation_output_audio_bytes / self._translation_output_bytes_per_second if self._translation_output_audio_bytes > 0 else None ) - resolved_output_seconds: Final = ( - output_seconds if isinstance(output_seconds, (int, float)) else synthetic_output_seconds - ) - if isinstance(input_seconds, (int, float)) or resolved_output_seconds is not None: + resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds + if input_seconds is not None or resolved_output_seconds is not None: if self._should_store_message(event_obj): - if not isinstance(output_seconds, (int, float)) and synthetic_output_seconds is not None: + if output_seconds is None and synthetic_output_seconds is not None: self.messages.append( OpenAIRealtimeTranslationClosedEvent( type="session.closed", @@ -588,10 +591,10 @@ class RealTimeStreaming: normalized_usage: Final = ( OpenAIRealtimeTranslationDurationUsage( type="duration", - input_seconds=float(input_seconds), + input_seconds=input_seconds, output_seconds=float(resolved_output_seconds or 0.0), ) - if isinstance(input_seconds, (int, float)) + if input_seconds is not None else OpenAIRealtimeTranslationDurationUsage( type="duration", output_seconds=float(resolved_output_seconds or 0.0) ) @@ -646,9 +649,9 @@ class RealTimeStreaming: if event.get("type") != "session.closed": continue event_usage = event.get("usage") # rebind-ok: each close event carries independent usage - if isinstance(event_usage, dict) and ( - isinstance(event_usage.get("input_seconds"), (int, float)) - or isinstance(event_usage.get("output_seconds"), (int, float)) + if ( + isinstance(event_usage, dict) + and normalized_audio_duration_seconds(event_usage.get("output_seconds")) is not None ): self._translation_usage_finalized = True return diff --git a/litellm/realtime_api/README.md b/litellm/realtime_api/README.md index d9699f57da7..d810de2f24f 100644 --- a/litellm/realtime_api/README.md +++ b/litellm/realtime_api/README.md @@ -6,16 +6,4 @@ Supported endpoints: Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI. -Billing visibility: -- WebSocket sessions pass provider usage events through LiteLLM and support local spend tracking -- Client-secret and SDP call endpoints only proxy session setup; subsequent WebRTC media and usage events travel over the peer connection, so LiteLLM cannot record inference spend or enforce spend-based budgets for those sessions -- Use the proxied WebSocket transport when LiteLLM spend logs and budgets must include Realtime inference - -Non-billable Realtime protocols are disabled by default. Operators who accept the billing and budget-enforcement limitation can opt in: - -```yaml -general_settings: - allow_non_billable_realtime_protocols: true -``` - -For user-facing documentation and usage examples, see the litellm-docs repo. +For user-facing documentation and usage examples, see the litellm-docs repo. \ No newline at end of file diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index db36ebc591c..6c79a873e0c 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3182,6 +3182,27 @@ def test_translation_retained_input_only_close_event_bills_captured_output(monke assert sum(usage.get("output_seconds", 0.0) for usage in usage_events) == 1.0 +@pytest.mark.parametrize("malformed_output", [-1.0, float("nan"), float("inf"), 10**1000]) +def test_translation_malformed_provider_duration_uses_captured_audio(malformed_output: float | int) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": malformed_output}} + ) + streaming._finalize_translation_usage() + + close_events: Final = tuple(event for event in streaming.messages if event.get("type") == "session.closed") + assert len(close_events) == 1 + assert close_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0} + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index f122a71d51a..e75bb6d3ed2 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10488,3 +10488,34 @@ def test_realtime_translation_duration_cost_includes_provider_input_usage( + (output_seconds or 0) * litellm.model_cost[model]["output_cost_per_second"] ) assert actual == pytest.approx(expected) + + +@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( + _local_model_cost_map, malformed_duration: float | int | bool +) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + input_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5}, + } + ] + output_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration}, + } + ] + + input_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + output_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + + assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"]) + assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) From 6ee09681c1036368149f9a16195f53afe1be65b9 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 04:34:03 -0500 Subject: [PATCH 14/45] fix(realtime): preserve batch cache rates and Azure v1 routes --- litellm/llms/azure/audio_transcriptions.py | 9 +++- ...odel_prices_and_context_window_backup.json | 24 ++++++++++ model_prices_and_context_window.json | 24 ++++++++++ .../llms/azure/test_audio_transcriptions.py | 44 +++++++++++++++++++ .../transcriptions/test_gpt_transcribe.py | 34 -------------- 5 files changed, 100 insertions(+), 35 deletions(-) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index caba5d48314..9e3d5b27b6c 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -43,8 +43,15 @@ class AzureAudioTranscription(AzureChatCompletion): ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) + model_info: Final = litellm.model_cost.get(f"azure/{model}") + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + requires_deployment_api: Final = model_info is None or ( + provider_specific_entry is not None and provider_specific_entry.get("transcription_deployment_api") == 1 + ) resolved_api_version: Final = ( - litellm.AZURE_DEFAULT_API_VERSION if api_version in ("v1", "latest", "preview") else api_version + litellm.AZURE_DEFAULT_API_VERSION + if requires_deployment_api and api_version in ("v1", "latest", "preview") + else api_version ) if atranscription is True: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6e3da13e476..29fc1ceef68 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -234,6 +234,7 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, + "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", @@ -29301,6 +29302,7 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -29410,6 +29412,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -29504,6 +29507,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -29602,6 +29606,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -29663,6 +29668,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -29721,6 +29727,7 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -30326,6 +30333,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -30386,6 +30394,7 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30446,6 +30455,7 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30506,6 +30516,7 @@ }, "vertex_ai/gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -32611,6 +32622,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -32671,6 +32683,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32731,6 +32744,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32791,6 +32805,7 @@ }, "gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -38480,6 +38495,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -38727,6 +38743,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -38827,6 +38844,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -52567,6 +52585,7 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -52638,6 +52657,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -52695,6 +52715,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -52793,6 +52814,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52855,6 +52877,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52914,6 +52937,7 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6e3da13e476..29fc1ceef68 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -234,6 +234,7 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, + "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", @@ -29301,6 +29302,7 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -29410,6 +29412,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -29504,6 +29507,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -29602,6 +29606,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -29663,6 +29668,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -29721,6 +29727,7 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -30326,6 +30333,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -30386,6 +30394,7 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30446,6 +30455,7 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -30506,6 +30516,7 @@ }, "vertex_ai/gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -32611,6 +32622,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -32671,6 +32683,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32731,6 +32744,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -32791,6 +32805,7 @@ }, "gemini-3.8-flash": { "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "cache_read_input_token_cost_priority": 1.35e-07, "google_maps_grounding_cost_per_query": 0.014, @@ -38480,6 +38495,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -38727,6 +38743,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -38827,6 +38844,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -52567,6 +52585,7 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -52638,6 +52657,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "cache_read_input_token_cost_priority": 9e-08, "deprecation_date": "2027-05-28", @@ -52695,6 +52715,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2027-06-28", @@ -52793,6 +52814,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -52855,6 +52877,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -52914,6 +52937,7 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, diff --git a/tests/unit/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py index c5d403b3c38..d8f1a290d5e 100644 --- a/tests/unit/llms/azure/test_audio_transcriptions.py +++ b/tests/unit/llms/azure/test_audio_transcriptions.py @@ -1,6 +1,8 @@ +import io import json from pathlib import Path from typing import Final +from unittest.mock import MagicMock import httpx import pytest @@ -9,6 +11,8 @@ from openai import AsyncAzureOpenAI, AzureOpenAI import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.types.utils import TranscriptionResponse AUDIO_FILE: Final = Path(__file__).parents[3] / "gettysburg.wav" WHISPER_COST_PER_SECOND: Final = 0.0001 @@ -160,3 +164,43 @@ async def test_azure_transcription_parses_response_formats( ) assert response.text == expected_text + + +@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) +@pytest.mark.parametrize( + ("model", "expected_path"), + [ + ("whisper-1", "/openai/v1/audio/transcriptions"), + ("gpt-transcribe", "/openai/deployments/gpt-transcribe/audio/transcriptions"), + ("custom-transcribe-deployment", "/openai/deployments/custom-transcribe-deployment/audio/transcriptions"), + ], +) +def test_azure_transcription_alias_uses_model_route( + monkeypatch: pytest.MonkeyPatch, model: str, expected_path: str, api_version: str +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == expected_path + assert request.url.params.get("api-version") == ( + None if expected_path.startswith("/openai/v1/") else litellm.AZURE_DEFAULT_API_VERSION + ) + return httpx.Response(200, json={"text": "hello"}) + + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with httpx.Client(transport=httpx.MockTransport(send_response)) as http_client: + monkeypatch.setattr(litellm, "client_session", http_client) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + response: Final = AzureAudioTranscription().audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="test-key", + api_base="https://example.openai.azure.com", + api_version=api_version, + ) + + assert response.text == "hello" diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index d192550943c..21660fca9f9 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -251,40 +251,6 @@ def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: No ) -@pytest.mark.parametrize( - ("api_version", "expected_api_version"), - [ - ("v1", litellm.AZURE_DEFAULT_API_VERSION), - ("latest", litellm.AZURE_DEFAULT_API_VERSION), - ("preview", litellm.AZURE_DEFAULT_API_VERSION), - (None, None), - ("2025-04-01-preview", "2025-04-01-preview"), - ], -) -@pytest.mark.parametrize("model", ["gpt-transcribe", "custom-transcribe-deployment"]) -def test_azure_audio_transcription_resolves_api_version_in_provider( - model: str, api_version: str | None, expected_api_version: str | None -) -> None: - handler = AzureAudioTranscription() - handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) - - handler.audio_transcriptions( - model=model, - audio_file=io.BytesIO(b"audio"), - optional_params={"stream": True}, - logging_obj=MagicMock(), - model_response=TranscriptionResponse(), - timeout=10, - max_retries=0, - api_key="sk-test", - api_base="https://example.openai.azure.com", - api_version=api_version, - atranscription=True, - ) - - assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == expected_api_version - - def test_azure_gpt_transcribe_uses_deployment_scoped_route(): def send_response(request: httpx.Request) -> httpx.Response: assert str(request.url) == ( From 7d1d0517501cf4803d48670aef3ea22c5aa2d930 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 04:49:54 -0500 Subject: [PATCH 15/45] fix(transcription): reject nonfinite provider durations --- litellm/cost_calculator.py | 4 +--- tests/unit/test_cost_calculator.py | 9 +++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index c236032dc9f..1b0eed1ed07 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1261,9 +1261,7 @@ def _get_transcription_usage_duration(completion_response: object) -> float | No seconds: Final = ( usage_object.get("seconds") if isinstance(usage_object, dict) else getattr(usage_object, "seconds", None) ) - if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds < 0: - return None - return float(seconds) + return normalized_audio_duration_seconds(seconds) def _extraction_usage(usage: ExtractionUsage) -> Usage: diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index e75bb6d3ed2..7625bb03191 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1143,6 +1143,15 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type(): assert _transcription_usage_cost({}, {}) == 0.0 +@pytest.mark.parametrize("malformed_seconds", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_transcription_ignores_invalid_provider_duration(malformed_seconds: float | int | bool) -> None: + from litellm.cost_calculator import _get_transcription_usage_duration + + response: Final = SimpleNamespace(usage={"type": "duration", "seconds": malformed_seconds}) + + assert _get_transcription_usage_duration(response) is None + + def test_get_transcription_model_falls_back_to_session_model(monkeypatch): """session.model is used when transcription-specific model fields are absent.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") From 099d9b3cc89416147d125ca057b169b184c8da31 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:03:44 -0500 Subject: [PATCH 16/45] refactor(realtime): align annotations with current lint rules --- litellm/litellm_core_utils/realtime_streaming.py | 3 +-- litellm/llms/custom_httpx/llm_http_handler.py | 2 +- .../llms/openai/transcriptions/gpt_transformation.py | 4 +--- litellm/llms/openai/transcriptions/handler.py | 8 +++++--- litellm/proxy/realtime_endpoints/endpoints.py | 10 +++------- litellm/realtime_api/main.py | 10 ++++++---- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 8d05ea17da0..d762082572a 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -648,9 +648,8 @@ class RealTimeStreaming: for event in self.messages: if event.get("type") != "session.closed": continue - event_usage = event.get("usage") # rebind-ok: each close event carries independent usage if ( - isinstance(event_usage, dict) + isinstance(event_usage := event.get("usage"), dict) and normalized_audio_duration_seconds(event_usage.get("output_seconds")) is not None ): self._translation_usage_finalized = True diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index dbde287b10a..cdca6ec3d4e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6874,7 +6874,7 @@ class BaseLLMHTTPHandler: configured_client: Final = openai_client.with_options( timeout=timeout, set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping - key: str(value) # mutable-ok: SDK headers are materialized as a concrete string mapping + key: str(value) for key, value in (extra_headers or {}).items() # mutable-ok: SDK requires concrete headers }, ) diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index 81f3abd217a..c55ecf1e56c 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -40,9 +40,7 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig): - def get_supported_openai_params( # mutable-ok: base transcription interface returns a mutable parameter list - self, model: str - ) -> list[OpenAIAudioTranscriptionOptionalParams]: + def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: return [ "prompt", "response_format", diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index f2d46a50ddd..16b8e919fc5 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -56,9 +56,11 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): try: sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: - response = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) - return None, response - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**sdk_data, timeout=timeout) + stream_response: Final = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) + return None, stream_response + raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( + **sdk_data, timeout=timeout + ) headers: Final = dict(raw_response.headers) response: Final = raw_response.parse() return headers, response diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 897f7a347f4..99d02fc7f4c 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -142,7 +142,7 @@ async def _authorize_and_bind_nested_transcription_models( session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization user_api_key_dict: UserAPIKeyAuth, llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list - llm_router: Any, + llm_router: "Router | None", ) -> None: nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data)) for nested_model in nested_models: @@ -522,18 +522,14 @@ async def proxy_realtime_calls( session_type = _coerce_realtime_session_type(raw_session_type) if is_translation_request != (raw_session_type == "translation"): return Response( - content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping - {"error": "Token is not valid for this Realtime endpoint"} - ), + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), status_code=http_status.HTTP_401_UNAUTHORIZED, media_type="application/json", ) else: if is_translation_request: return Response( - content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping - {"error": "Token is not valid for this Realtime endpoint"} - ), + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), status_code=http_status.HTTP_401_UNAUTHORIZED, media_type="application/json", ) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 969d49b37dc..b9e0b655cc3 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -83,12 +83,14 @@ def _with_resolved_session_model(session: dict[str, object], model_name: str) -> audio = session.get("audio") audio = audio if isinstance(audio, dict) else {} # mutable-ok: nested session model is rebuilt locally audio_input = audio.get("input") - audio_input = ( # mutable-ok: nested session model is rebuilt locally - audio_input if isinstance(audio_input, dict) else {} + audio_input = ( + audio_input if isinstance(audio_input, dict) else {} # mutable-ok: nested session model is rebuilt locally ) transcription = audio_input.get("transcription") - transcription = ( # mutable-ok: nested session model is rebuilt locally - transcription if isinstance(transcription, dict) else {} + transcription = ( + transcription + if isinstance(transcription, dict) + else {} # mutable-ok: nested session model is rebuilt locally ) return { # mutable-ok: provider routing requires an independently mutable session payload **session, From eaaccfcc5ad4d3e0d9bc1a3dd40f418cb62b1129 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:24:14 -0500 Subject: [PATCH 17/45] fix(realtime): preserve translation deployment pricing and input usage --- litellm/cost_calculator.py | 36 +++++---- .../litellm_core_utils/realtime_streaming.py | 77 +++++++++++++++---- tests/unit/cookbook/__init__.py | 0 .../test_realtime_streaming.py | 56 ++++++++++++++ 4 files changed, 139 insertions(+), 30 deletions(-) create mode 100644 tests/unit/cookbook/__init__.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 1b0eed1ed07..9ce1be1a41a 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -3557,6 +3557,7 @@ def handle_realtime_stream_cost_calculation( results=results, custom_llm_provider=custom_llm_provider, litellm_model_name=litellm_model_name, + potential_model_names=potential_model_names, ) live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"}) live_session_started: Final = any(result["type"] == "session.started" for result in results) @@ -3645,6 +3646,7 @@ def handle_realtime_translation_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, litellm_model_name: str, + potential_model_names: Sequence[str | None] = (), ) -> float: usage_events: Final = tuple( usage @@ -3664,22 +3666,28 @@ def handle_realtime_translation_cost_calculation( ) if input_seconds <= 0 and output_seconds <= 0: return 0.0 - try: - model_info: Final = litellm.get_model_info( - model=litellm_model_name, - custom_llm_provider=custom_llm_provider, - ) - except Exception: # noqa: BLE001 # unknown model metadata should yield zero translation cost - return 0.0 - input_cost_per_second: Final = model_info.get("input_cost_per_second") - output_cost_per_second: Final = model_info.get("output_cost_per_second") - input_cost: Final = ( - input_seconds * input_cost_per_second if isinstance(input_cost_per_second, (int, float)) else 0.0 + model_infos: Final = tuple( + _get_model_info_or_none(model, custom_llm_provider) + for model in (*potential_model_names, litellm_model_name) + if model is not None ) - output_cost: Final = ( - output_seconds * output_cost_per_second if isinstance(output_cost_per_second, (int, float)) else 0.0 + input_cost_per_second: Final = next( + ( + rate + for info in model_infos + if (rate := _declared_transcription_rate(info, ("input_cost_per_second",))) is not None + ), + 0.0, ) - return input_cost + output_cost + output_cost_per_second: Final = next( + ( + rate + for info in model_infos + if (rate := _declared_transcription_rate(info, ("output_cost_per_second",))) is not None + ), + 0.0, + ) + return input_seconds * input_cost_per_second + output_seconds * output_cost_per_second def handle_realtime_transcription_cost_calculation( diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index d762082572a..aaa29b275b5 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -260,6 +260,8 @@ class RealTimeStreaming: self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] self._is_translation_session = translation_session + self._translation_input_seconds = 0.0 + self._translation_input_bytes_per_second = 48000.0 self._translation_output_audio_bytes = 0 self._translation_output_bytes_per_second = 48000.0 self._translation_usage_finalized = False @@ -620,27 +622,61 @@ class RealTimeStreaming: return self._translation_output_audio_bytes += len(decoded) + @staticmethod + def _translation_audio_bytes_per_second(audio_format: object) -> float | None: + if audio_format == "pcm16": + return 48000.0 + if audio_format in ("g711_ulaw", "g711_alaw"): + return 8000.0 + if not isinstance(audio_format, Mapping): + return None + rate: Final = normalized_audio_duration_seconds(audio_format.get("rate")) + if rate is None or rate <= 0: + return None + match audio_format.get("type"): + case "audio/pcm": + return rate * 2 + case "audio/pcmu" | "audio/pcma": + return rate + case _: + return None + + def _capture_translation_input_audio(self, message: str) -> None: + if not self._is_translation_session: + return + try: + event: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return + self._capture_translation_output_format(event) + if event.get("type") not in ( + "input_audio_buffer.append", + "session.input_audio_buffer.append", + ) or not isinstance(audio := event.get("audio"), str): + return + try: + decoded: Final = base64.b64decode(audio, validate=True) + except (ValueError, TypeError): + return + self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second + def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: session: Final = event_obj.get("session") if not isinstance(session, dict): return audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + input_format: Final = ( + audio_input.get("format") if isinstance(audio_input, dict) else session.get("input_audio_format") + ) + input_rate: Final = self._translation_audio_bytes_per_second(input_format) + if input_rate is not None: + self._translation_input_bytes_per_second = input_rate output: Final = audio.get("output") if isinstance(audio, dict) else None audio_format: Final = output.get("format") if isinstance(output, dict) else None - if isinstance(audio_format, str): - if audio_format in ("g711_ulaw", "g711_alaw"): - self._translation_output_bytes_per_second = 8000.0 - return - if not isinstance(audio_format, dict): - return - format_type: Final = audio_format.get("type") - rate: Final = audio_format.get("rate") - if not isinstance(rate, (int, float)) or rate <= 0: - return - if format_type == "audio/pcm": - self._translation_output_bytes_per_second = float(rate) * 2 - elif format_type in ("audio/pcmu", "audio/pcma"): - self._translation_output_bytes_per_second = float(rate) + output_rate: Final = self._translation_audio_bytes_per_second(audio_format) + if output_rate is not None: + self._translation_output_bytes_per_second = output_rate def _finalize_translation_usage(self) -> None: if self._translation_usage_finalized: @@ -654,10 +690,16 @@ class RealTimeStreaming: ): self._translation_usage_finalized = True return - if self._translation_output_audio_bytes == 0: + if self._translation_output_audio_bytes == 0 and self._translation_input_seconds == 0: return output_seconds: Final = self._translation_output_audio_bytes / self._translation_output_bytes_per_second - synthetic_usage: Final = OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + synthetic_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", input_seconds=self._translation_input_seconds, output_seconds=output_seconds + ) + if self._translation_input_seconds > 0 + else OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + ) self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) self._translation_usage_finalized = True @@ -711,12 +753,15 @@ class RealTimeStreaming: if is_content_message: self._content_sent_after_setup = True sent = True + if sent: + self._capture_translation_input_audio(message) return sent await self.backend_ws.send(message) if self._usage_meter is not None: sent_event: Final = self._parse_backend_event(message) if sent_event is not None: self._usage_meter.observe_client_event(sent_event) + self._capture_translation_input_audio(message) return True async def _apply_nested_transcription_model_policy(self, message: str) -> str: diff --git a/tests/unit/cookbook/__init__.py b/tests/unit/cookbook/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 6c79a873e0c..cfe470c7489 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3066,6 +3066,62 @@ def test_translation_audio_duration_is_finalized_once(event_type: str): assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 1.0} +@pytest.mark.asyncio +@pytest.mark.parametrize("output_bytes", (0, 48000)) +@pytest.mark.parametrize("event_type", ("input_audio_buffer.append", "session.input_audio_buffer.append")) +@pytest.mark.parametrize( + "audio_format,bytes_per_second", + [("pcm16", 48000), ("g711_ulaw", 8000), ({"type": "audio/pcm", "rate": 16000}, 32000)], +) +async def test_translation_disconnect_bills_sent_input_audio( + output_bytes: int, event_type: str, audio_format: str | Mapping[str, object], bytes_per_second: int +) -> None: + import base64 + + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), + backend_ws=backend, + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + await streaming._send_to_backend( + json.dumps({"type": "session.update", "session": {"audio": {"input": {"format": audio_format}}}}) + ) + await streaming._send_to_backend( + json.dumps({"type": event_type, "audio": base64.b64encode(bytes(2 * bytes_per_second)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} + ) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + assert streaming.messages == [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": output_bytes / 48000}, + } + ] + + +@pytest.mark.asyncio +async def test_translation_failed_audio_send_is_not_billed() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock(side_effect=RuntimeError("send failed")) + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + + with pytest.raises(RuntimeError, match="send failed"): + await streaming._send_to_backend(json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})) + streaming._finalize_translation_usage() + + assert streaming.messages == [] + + def test_translation_audio_duration_uses_session_output_format(): import base64 From b06e6d269543dcf515d6cfb1af003a199200db03 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:29:37 -0500 Subject: [PATCH 18/45] test(transcription): verify synchronous stream finalization --- .../transcriptions/test_gpt_transcribe.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 21660fca9f9..535b52a17ee 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -3,11 +3,12 @@ import json import wave from collections.abc import Iterator from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest -from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI import litellm from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream @@ -66,6 +67,39 @@ def test_transcription_response_preserves_empty_languages(): assert response.model_dump()["languages"] == [] +def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + 'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + 'data: {"type":"transcript.text.done","text":"hello world",' + '"usage":{"type":"duration","seconds":2.5}}\n\n' + ), + ) + + logging_obj: Final = MagicMock() + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + stream: Final = client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + wrapped: Final = wrap_transcription_stream(stream, logging_obj, datetime(2026, 1, 1)) + received: Final = tuple(wrapped) + wrapped.close() + + assert tuple(event.type for event in received) == ("transcript.text.delta", "transcript.text.done") + logging_obj.success_handler.assert_called_once() + logged_response: Final = logging_obj.success_handler.call_args.args[0] + assert logged_response.text == "hello world" + assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5} + logging_obj.failure_handler.assert_not_called() + + @pytest.mark.asyncio async def test_openai_handler_returns_native_typed_stream(): async def send_response(request: httpx.Request) -> httpx.Response: From 0e69d04d36fccc63f3d22f8963b4be1ea58706b2 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:36:33 -0500 Subject: [PATCH 19/45] fix(realtime): retain missing translation input usage at close --- .../litellm_core_utils/realtime_streaming.py | 73 ++++++++----------- .../test_realtime_streaming.py | 48 +++++++++--- 2 files changed, 69 insertions(+), 52 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index aaa29b275b5..9ed257d9f42 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -261,7 +261,6 @@ class RealTimeStreaming: self.tool_calls: list[dict] = [] self._is_translation_session = translation_session self._translation_input_seconds = 0.0 - self._translation_input_bytes_per_second = 48000.0 self._translation_output_audio_bytes = 0 self._translation_output_bytes_per_second = 48000.0 self._translation_usage_finalized = False @@ -569,9 +568,14 @@ class RealTimeStreaming: output_seconds: Final = ( normalized_audio_duration_seconds(usage.get("output_seconds")) if isinstance(usage, dict) else None ) - input_seconds: Final = ( + reported_input_seconds: Final = ( normalized_audio_duration_seconds(usage.get("input_seconds")) if isinstance(usage, dict) else None ) + input_seconds: Final = ( + reported_input_seconds + if reported_input_seconds is not None + else self._translation_input_seconds or None + ) synthetic_output_seconds: Final = ( self._translation_output_audio_bytes / self._translation_output_bytes_per_second if self._translation_output_audio_bytes > 0 @@ -580,13 +584,19 @@ class RealTimeStreaming: resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds if input_seconds is not None or resolved_output_seconds is not None: if self._should_store_message(event_obj): - if output_seconds is None and synthetic_output_seconds is not None: + supplemental_usage: Final = OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=float(input_seconds or 0.0) if reported_input_seconds is None else 0.0, + output_seconds=float(synthetic_output_seconds or 0.0) if output_seconds is None else 0.0, + ) + if ( + supplemental_usage.get("input_seconds", 0.0) > 0 + or supplemental_usage.get("output_seconds", 0.0) > 0 + ): self.messages.append( OpenAIRealtimeTranslationClosedEvent( type="session.closed", - usage=OpenAIRealtimeTranslationDurationUsage( - type="duration", output_seconds=synthetic_output_seconds - ), + usage=supplemental_usage, ) ) else: @@ -622,25 +632,6 @@ class RealTimeStreaming: return self._translation_output_audio_bytes += len(decoded) - @staticmethod - def _translation_audio_bytes_per_second(audio_format: object) -> float | None: - if audio_format == "pcm16": - return 48000.0 - if audio_format in ("g711_ulaw", "g711_alaw"): - return 8000.0 - if not isinstance(audio_format, Mapping): - return None - rate: Final = normalized_audio_duration_seconds(audio_format.get("rate")) - if rate is None or rate <= 0: - return None - match audio_format.get("type"): - case "audio/pcm": - return rate * 2 - case "audio/pcmu" | "audio/pcma": - return rate - case _: - return None - def _capture_translation_input_audio(self, message: str) -> None: if not self._is_translation_session: return @@ -648,35 +639,35 @@ class RealTimeStreaming: event: Final = _decode_json_object(message) except (json.JSONDecodeError, TypeError): return - self._capture_translation_output_format(event) - if event.get("type") not in ( - "input_audio_buffer.append", - "session.input_audio_buffer.append", - ) or not isinstance(audio := event.get("audio"), str): + if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str): return try: decoded: Final = base64.b64decode(audio, validate=True) except (ValueError, TypeError): return - self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second + self._translation_input_seconds += len(decoded) / 48000.0 def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: session: Final = event_obj.get("session") if not isinstance(session, dict): return audio: Final = session.get("audio") - audio_input: Final = audio.get("input") if isinstance(audio, dict) else None - input_format: Final = ( - audio_input.get("format") if isinstance(audio_input, dict) else session.get("input_audio_format") - ) - input_rate: Final = self._translation_audio_bytes_per_second(input_format) - if input_rate is not None: - self._translation_input_bytes_per_second = input_rate output: Final = audio.get("output") if isinstance(audio, dict) else None audio_format: Final = output.get("format") if isinstance(output, dict) else None - output_rate: Final = self._translation_audio_bytes_per_second(audio_format) - if output_rate is not None: - self._translation_output_bytes_per_second = output_rate + if isinstance(audio_format, str): + if audio_format in ("g711_ulaw", "g711_alaw"): + self._translation_output_bytes_per_second = 8000.0 + return + if not isinstance(audio_format, dict): + return + format_type: Final = audio_format.get("type") + rate: Final = audio_format.get("rate") + if not isinstance(rate, (int, float)) or rate <= 0: + return + if format_type == "audio/pcm": + self._translation_output_bytes_per_second = float(rate) * 2 + elif format_type in ("audio/pcmu", "audio/pcma"): + self._translation_output_bytes_per_second = float(rate) def _finalize_translation_usage(self) -> None: if self._translation_usage_finalized: diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index cfe470c7489..267e95d2773 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3068,13 +3068,8 @@ def test_translation_audio_duration_is_finalized_once(event_type: str): @pytest.mark.asyncio @pytest.mark.parametrize("output_bytes", (0, 48000)) -@pytest.mark.parametrize("event_type", ("input_audio_buffer.append", "session.input_audio_buffer.append")) -@pytest.mark.parametrize( - "audio_format,bytes_per_second", - [("pcm16", 48000), ("g711_ulaw", 8000), ({"type": "audio/pcm", "rate": 16000}, 32000)], -) async def test_translation_disconnect_bills_sent_input_audio( - output_bytes: int, event_type: str, audio_format: str | Mapping[str, object], bytes_per_second: int + output_bytes: int, ) -> None: import base64 @@ -3088,10 +3083,7 @@ async def test_translation_disconnect_bills_sent_input_audio( translation_session=True, ) await streaming._send_to_backend( - json.dumps({"type": "session.update", "session": {"audio": {"input": {"format": audio_format}}}}) - ) - await streaming._send_to_backend( - json.dumps({"type": event_type, "audio": base64.b64encode(bytes(2 * bytes_per_second)).decode()}) + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) ) streaming._capture_translation_output_audio( {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} @@ -3116,12 +3108,46 @@ async def test_translation_failed_audio_send_is_not_billed() -> None: ) with pytest.raises(RuntimeError, match="send failed"): - await streaming._send_to_backend(json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})) + await streaming._send_to_backend(json.dumps({"type": "session.input_audio_buffer.append", "audio": "AAAA"})) streaming._finalize_translation_usage() assert streaming.messages == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("retain_close", (False, True)) +@pytest.mark.parametrize("reported_input,expected_input", [(None, 2.0), (0.0, 0.0), (0.25, 0.25)]) +async def test_translation_terminal_usage_fills_only_missing_input_duration( + monkeypatch: pytest.MonkeyPatch, retain_close: bool, reported_input: float | None, expected_input: float +) -> None: + import base64 + + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*" if retain_close else None) + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) + ) + close_event: Final = { + "type": "session.closed", + "usage": { + "type": "duration", + "output_seconds": 0.5, + **({"input_seconds": reported_input} if reported_input is not None else {}), + }, + } + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert sum(item.get("input_seconds") or 0.0 for item in usage) == expected_input + assert sum(item.get("output_seconds") or 0.0 for item in usage) == 0.5 + + def test_translation_audio_duration_uses_session_output_format(): import base64 From 913c5dd2505446839b5e67b7aa17c8ad14d6a039 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 10:01:14 -0500 Subject: [PATCH 20/45] fix(transcription): reject streams that bypass output guardrails --- litellm/proxy/proxy_server.py | 6 +++ litellm/proxy/utils.py | 11 ++++ .../proxy/proxy_server/test_routes_audio.py | 51 +++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 668cb15ac65..adda6f1ded0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13430,6 +13430,12 @@ async def audio_transcriptions( call_type="transcription", ) + if data.get("stream") is True and ProxyLogging.has_post_call_guardrails(data, llm_router): + raise HTTPException( + detail="Streaming transcription does not support output guardrails. Use stream=false.", + status_code=status.HTTP_400_BAD_REQUEST, + ) + ## ROUTE TO CORRECT ENDPOINT ## llm_call: Final = await route_request( data=data, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 28be11509ae..b29bec0bad5 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3333,6 +3333,17 @@ class ProxyLogging: def has_post_call_response_headers_callbacks() -> bool: return ProxyLogging.callback_capabilities().has_post_call_response_headers + @staticmethod + def has_post_call_guardrails(request_data: Mapping[str, object], llm_router: Router | None) -> bool: + guardrail_data: Final = _check_and_merge_model_level_guardrails( + data=dict(request_data), llm_router=llm_router, trust_client_model_info=False + ) + guardrails, _ = _partition_post_call_callbacks() + return bool(pipeline_managed_guardrail_names(guardrail_data, "post_call")) or any( + guardrail.should_run_guardrail(data=guardrail_data, event_type=GuardrailEventHooks.post_call) + for guardrail in guardrails + ) + @staticmethod def has_streaming_callbacks() -> bool: caps: Final = ProxyLogging.callback_capabilities() diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index a756454405e..03cfe3fa67e 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -10,13 +10,21 @@ Pins (PR2): from __future__ import annotations import io +from collections.abc import Callable +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi.testclient import TestClient +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep @pytest.fixture @@ -305,3 +313,46 @@ def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcr assert response.headers["content-type"].startswith("text/event-stream") assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\n' assert patched_transcription_stream.closed is True + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize( + "configuration,expected_status", + [("default", 400), ("model", 400), ("policy", 400), ("pre_call", 200), ("disabled", 200)], +) +def test_streaming_transcription_rejects_applicable_output_guardrails( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + configuration: str, + expected_status: int, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="transcription-output", + event_hook=GuardrailEventHooks.pre_call if configuration == "pre_call" else GuardrailEventHooks.post_call, + default_on=configuration in ("default", "pre_call"), + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + proxy_server.llm_router.get_model_list.return_value = ( + [{"litellm_params": {"guardrails": ["transcription-output"]}}] if configuration == "model" else [] + ) + if configuration == "policy": + pipeline: Final = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="transcription-output")]) + proxy_server.proxy_logging_obj.pre_call_hook.side_effect = lambda **kwargs: { + **kwargs["data"], + "metadata": {"_guardrail_pipelines": [("transcription-policy", pipeline)]}, + } + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == expected_status + if expected_status == 400: + assert "stream=false" in response.json()["error"]["message"] + assert "hello world" not in response.text + else: + assert '"text":"hello world"' in response.text From 515f6da5980c8b757f68885714590aa3e6eb0280 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 09:24:20 -0500 Subject: [PATCH 21/45] fix(realtime): reconcile audio protocols billing and current CI --- .github/workflows/test-unit.yml | 1 + .../audio_utils/transcription_streaming.py | 8 +-- .../litellm_core_utils/realtime_streaming.py | 67 +++++++++++++------ .../azure/realtime/http_transformation.py | 2 +- litellm/llms/custom_httpx/llm_http_handler.py | 19 ++---- litellm/llms/openai/realtime/handler.py | 6 +- .../transcriptions/gpt_transformation.py | 4 +- litellm/llms/openai/transcriptions/handler.py | 2 +- litellm/main.py | 8 ++- litellm/proxy/_lazy_features.py | 2 +- .../auth/managed_authorization.py | 11 ++- .../proxy/common_utils/http_parsing_utils.py | 6 +- litellm/proxy/proxy_server.py | 4 +- litellm/proxy/realtime_endpoints/endpoints.py | 18 ++--- litellm/realtime_api/main.py | 40 +++++------ .../test_realtime_streaming.py | 43 ++++++++++-- .../realtime/test_openai_realtime_handler.py | 53 +++++++++------ .../auth/test_managed_authorization.py | 17 ++++- 18 files changed, 193 insertions(+), 118 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2276efc4e34..1f7c170d329 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -422,6 +422,7 @@ jobs: tests/unit/ocr tests/unit/passthrough tests/unit/rag + tests/unit/cookbook tests/unit/realtime_api tests/unit/rerank_api tests/unit/search diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 979a1847362..cf2681f5d36 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -72,13 +72,7 @@ class _TranscriptionEventCollector: response: Final = TranscriptionResponse( text=done_event.text if done_event is not None else "".join(self.text_deltas), usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, - languages=( - [ # mutable-ok: the response model requires a concrete serialized language list - language.model_dump() for language in done_languages - ] - if done_languages is not None - else None - ), + languages=([language.model_dump() for language in done_languages] if done_languages is not None else None), ) if self.duration is not None: response.set_audio_transcription_duration(self.duration) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 9ed257d9f42..04fed504038 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -8,7 +8,7 @@ from enum import Enum, auto from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly import litellm @@ -207,6 +207,25 @@ def _gate_session_update( return {**client_session, "turn_detection": gate(None)} +_TRANSLATION_AUDIO_FORMAT: Final = TypeAdapter(str | Mapping[str, object] | None) + + +def _translation_audio_bytes_per_second(audio_format: str | Mapping[str, object] | None) -> float | None: + if isinstance(audio_format, str): + return 8000.0 if audio_format in ("g711_ulaw", "g711_alaw") else None + if audio_format is None: + return None + format_type: Final = audio_format.get("type") + rate: Final = audio_format.get("rate") + if format_type in ("audio/pcmu", "audio/pcma") and rate is None: + return 8000.0 + if not isinstance(rate, (int, float)) or rate <= 0: + return None + if format_type == "audio/pcm": + return float(rate) * 2 + return float(rate) if format_type in ("audio/pcmu", "audio/pcma") else None + + class RealtimeEventNormalizer(Protocol): def should_drop(self, event: object) -> bool: ... def normalize(self, event: dict) -> dict: ... @@ -261,13 +280,14 @@ class RealTimeStreaming: self.tool_calls: list[dict] = [] self._is_translation_session = translation_session self._translation_input_seconds = 0.0 + self._translation_input_bytes_per_second = 48000.0 self._translation_output_audio_bytes = 0 self._translation_output_bytes_per_second = 48000.0 self._translation_usage_finalized = False # Detect whether the client is explicitly opting into the beta protocol. self._client_wants_beta = self.detect_beta_header(websocket) - self._backend_uses_beta_protocol = ( + self._backend_uses_beta_protocol = not translation_session and ( self._client_wants_beta if backend_uses_beta_protocol is None else backend_uses_beta_protocol ) @@ -349,8 +369,8 @@ class RealTimeStreaming: _CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"]) _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, - "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, - "g711_alaw": {"type": "audio/G711-alaw", "rate": 8000}, + "g711_ulaw": {"type": "audio/pcmu"}, + "g711_alaw": {"type": "audio/pcma"}, } # GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1) _GA_TO_BETA_EVENT_TYPES: dict[str, str] = { @@ -616,7 +636,7 @@ class RealTimeStreaming: ) self._translation_usage_finalized = True return - self._capture_translation_output_format(event_obj) + self._capture_translation_audio_formats(event_obj) if event_obj.get("type") not in ( "session.output_audio.delta", "response.output_audio.delta", @@ -639,35 +659,38 @@ class RealTimeStreaming: event: Final = _decode_json_object(message) except (json.JSONDecodeError, TypeError): return + self._capture_translation_audio_formats(event) if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str): return try: decoded: Final = base64.b64decode(audio, validate=True) except (ValueError, TypeError): return - self._translation_input_seconds += len(decoded) / 48000.0 + self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second - def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: + def _capture_translation_audio_formats(self, event_obj: Mapping[str, object]) -> None: session: Final = event_obj.get("session") if not isinstance(session, dict): return audio: Final = session.get("audio") - output: Final = audio.get("output") if isinstance(audio, dict) else None - audio_format: Final = output.get("format") if isinstance(output, dict) else None - if isinstance(audio_format, str): - if audio_format in ("g711_ulaw", "g711_alaw"): - self._translation_output_bytes_per_second = 8000.0 + if not isinstance(audio, dict): return - if not isinstance(audio_format, dict): - return - format_type: Final = audio_format.get("type") - rate: Final = audio_format.get("rate") - if not isinstance(rate, (int, float)) or rate <= 0: - return - if format_type == "audio/pcm": - self._translation_output_bytes_per_second = float(rate) * 2 - elif format_type in ("audio/pcmu", "audio/pcma"): - self._translation_output_bytes_per_second = float(rate) + for direction, configuration in (("input", audio.get("input")), ("output", audio.get("output"))): + if not isinstance(configuration, dict): + continue + try: + if ( + bytes_per_second := _translation_audio_bytes_per_second( + _TRANSLATION_AUDIO_FORMAT.validate_python(configuration.get("format")) + ) + ) is None: + continue + except ValidationError: + continue + if direction == "input": + self._translation_input_bytes_per_second = bytes_per_second + else: + self._translation_output_bytes_per_second = bytes_per_second def _finalize_translation_usage(self) -> None: if self._translation_usage_finalized: diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 285d7f487ff..732b0946c6d 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -52,7 +52,7 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): model: str, api_key: str | None = None, ) -> dict: - validated_headers: Final = { # mutable-ok: provider authentication headers are extended before dispatch + validated_headers: Final = { **headers, "Content-Type": "application/json", } diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index cdca6ec3d4e..1365b9d30ab 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6865,7 +6865,7 @@ class BaseLLMHTTPHandler: logging_obj.pre_call( input=request_data, api_key="", - additional_args={ # mutable-ok: logging owns a mutable request metadata payload + additional_args={ "complete_input_dict": request_data, "api_base": normalized_api_base, }, @@ -6873,10 +6873,7 @@ class BaseLLMHTTPHandler: try: configured_client: Final = openai_client.with_options( timeout=timeout, - set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping - key: str(value) - for key, value in (extra_headers or {}).items() # mutable-ok: SDK requires concrete headers - }, + set_default_headers={key: str(value) for key, value in (extra_headers or {}).items()}, ) raw_response: Final = await configured_client.post( "/realtime/client_secrets", @@ -6948,7 +6945,7 @@ class BaseLLMHTTPHandler: logging_obj.pre_call( input=request_data, api_key="", - additional_args={ # mutable-ok: logging owns a mutable request metadata payload + additional_args={ "complete_input_dict": request_data, "api_base": normalized_api_base, }, @@ -7081,7 +7078,7 @@ class BaseLLMHTTPHandler: logging_obj.pre_call( input="realtime_sdp_offer", api_key="", - additional_args={ # mutable-ok: logging owns a mutable request metadata payload + additional_args={ "api_base": normalized_api_base, "session": session_data, }, @@ -7090,7 +7087,7 @@ class BaseLLMHTTPHandler: if translation: configured_client: Final = openai_client.with_options( timeout=timeout, - set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + set_default_headers={ "Content-Type": "application/sdp", **MappingProxyType( {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} @@ -7131,7 +7128,7 @@ class BaseLLMHTTPHandler: @staticmethod def _decoded_realtime_sdk_response(response: httpx.Response) -> httpx.Response: - headers: Final = { # mutable-ok: httpx accepts a concrete response header mapping + headers: Final = { key: value for key, value in response.headers.items() if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") @@ -7176,9 +7173,7 @@ class BaseLLMHTTPHandler: - session: JSON string with {"type": "realtime", "model": "...", ...} """ session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally - **( - session_config or {} # mutable-ok: absent session configuration starts from an empty provider payload - ) + **(session_config or {}) } if "type" not in session_data: session_data["type"] = "translation" if translation else "realtime" diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 1faebd4bb64..259a920b975 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -212,7 +212,7 @@ class OpenAIRealtime(OpenAIChatCompletion): model=sdk_model, extra_query=extra_query, extra_headers=headers, - websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping + websocket_connection_options={ "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, **(MappingProxyType({"ssl": ssl_config}) if url.startswith("wss://") else MappingProxyType({})), **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), @@ -253,7 +253,9 @@ class OpenAIRealtime(OpenAIChatCompletion): # Get provider-specific SSL configuration ssl_config: Final = self._get_ssl_config(url) - openai_beta_realtime: Final = client_sent_openai_beta_realtime_header(websocket) + openai_beta_realtime: Final = realtime_mode != "translation" and client_sent_openai_beta_realtime_header( + websocket + ) if not openai_beta_realtime: verbose_logger.debug( "OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). " diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index c55ecf1e56c..cbeb7a6d811 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -14,7 +14,7 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): """ Get the supported OpenAI params for the `gpt-4o-transcribe` models """ - return [ # mutable-ok: base transcription interface requires a mutable supported-parameter list + return [ "language", "prompt", "response_format", @@ -56,7 +56,7 @@ class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionCon optional_params: dict, # mutable-ok: base transformation interface supplies a mutable request payload litellm_params: dict, # mutable-ok: base transformation interface supplies mutable provider parameters ) -> AudioTranscriptionRequestData: - data: Final = { # mutable-ok: OpenAI SDK consumes this multipart request mapping + data: Final = { "model": model, "file": audio_file, "response_format": "json", diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 16b8e919fc5..8f020901714 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -36,7 +36,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout) - return {}, stream_response # mutable-ok: response headers use the existing mutable mapping contract + return {}, stream_response raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( **sdk_data, timeout=timeout ) # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options diff --git a/litellm/main.py b/litellm/main.py index 1f12b933bee..09880ec5768 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8218,7 +8218,9 @@ async def atranscription( if file is not None: calculated_duration = calculate_request_duration(file) if calculated_duration is not None: - response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field + response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it + calculated_duration + ) return response if not isinstance(response, TranscriptionResponse): raise ValueError( @@ -8567,7 +8569,9 @@ def transcription( calculated_duration: Final = calculate_request_duration(file) if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field + response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it + calculated_duration + ) elif getattr(response, "duration", None) is None and calculated_duration is not None: response.set_audio_transcription_duration(calculated_duration) diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 9942d6ea113..0feb32592d8 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -47,7 +47,7 @@ def _include_router_prepend(attr_name: str = "router") -> Callable[["FastAPI", o route_count: Final = len(router.routes) app.include_router(getattr(module, attr_name)) new_routes: Final = router.routes[route_count:] - router.routes[:] = [ # mutable-ok: FastAPI route registration requires in-place list replacement + router.routes[:] = [ *new_routes, *router.routes[:route_count], ] diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index cdbbd6029fa..229c5d23d74 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -11,7 +11,16 @@ from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failur from litellm.types.agents import AgentResponse from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext -_MANAGED_REALTIME_ROUTES: Final = frozenset(("/realtime", "/v1/realtime", "/openai/v1/realtime")) +_MANAGED_REALTIME_ROUTES: Final = frozenset( + ( + "/realtime", + "/v1/realtime", + "/openai/v1/realtime", + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", + ) +) _MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions")) _MANAGED_MODEL_ROUTES: Final = frozenset( f"{prefix}/{operation}" diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 56a46de67f2..5d4dcda1e83 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -527,10 +527,8 @@ async def get_form_data(request: Request) -> dict[str, Any]: array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]")) normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items) normalized_keys: Final = frozenset(key for key, _ in normalized_items) - return { # mutable-ok: request parsers expose a mutable form-data mapping to endpoint handlers - key: [ # mutable-ok: repeated multipart values follow the established mutable list contract - value for item_key, value in normalized_items if item_key == key - ] + return { + key: [value for item_key, value in normalized_items if item_key == key] if key in array_keys else next(value for item_key, value in reversed(normalized_items) if item_key == key) for key in normalized_keys diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index adda6f1ded0..d2c1d5c9268 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13754,9 +13754,7 @@ async def _route_realtime_websocket_session( ) query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys RealtimeQueryParams, - dict( # mutable-ok: downstream realtime routing normalizes this request-scoped query mapping - _realtime_query_params_template(query_model, intent) - ), + dict(_realtime_query_params_template(query_model, intent)), ) data: dict[str, object] = { diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 99d02fc7f4c..c28439a6395 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -289,18 +289,18 @@ def _decode_realtime_token_payload( ) @router.post( "/v1/realtime/translations/client_secrets", - dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists - tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], ) @router.post( "/realtime/translations/client_secrets", - dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists - tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], ) @router.post( "/openai/v1/realtime/translations/client_secrets", - dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists - tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], ) async def create_realtime_client_secret( request: Request, @@ -453,15 +453,15 @@ async def create_realtime_client_secret( ) @router.post( "/v1/realtime/translations/calls", - tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists + tags=["realtime"], ) @router.post( "/realtime/translations/calls", - tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists + tags=["realtime"], ) @router.post( "/openai/v1/realtime/translations/calls", - tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists + tags=["realtime"], ) async def proxy_realtime_calls( request: Request, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index b9e0b655cc3..de807867bed 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -81,24 +81,18 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: if session.get("type") == "transcription": audio = session.get("audio") - audio = audio if isinstance(audio, dict) else {} # mutable-ok: nested session model is rebuilt locally + audio = audio if isinstance(audio, dict) else {} audio_input = audio.get("input") - audio_input = ( - audio_input if isinstance(audio_input, dict) else {} # mutable-ok: nested session model is rebuilt locally - ) + audio_input = audio_input if isinstance(audio_input, dict) else {} transcription = audio_input.get("transcription") - transcription = ( - transcription - if isinstance(transcription, dict) - else {} # mutable-ok: nested session model is rebuilt locally - ) - return { # mutable-ok: provider routing requires an independently mutable session payload + transcription = transcription if isinstance(transcription, dict) else {} + return { **session, - "audio": { # mutable-ok: provider routing rebuilds nested audio configuration + "audio": { **audio, - "input": { # mutable-ok: provider routing rebuilds nested input configuration + "input": { **audio_input, - "transcription": { # mutable-ok: resolved deployment replaces only the transcription model + "transcription": { **transcription, "model": model_name, }, @@ -248,7 +242,7 @@ async def acreate_realtime_client_secret( if isinstance(request_data.get("session"), dict): request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) elif req.model is not None: - request_data["session"] = { # mutable-ok: OpenAI SDK consumes this request-scoped session payload + request_data["session"] = { "type": "realtime", "model": model_name, } @@ -279,7 +273,7 @@ async def acreate_realtime_translation_client_secret( ) -> httpx.Response: requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate" session_config: Final = RealtimeSessionConfig.model_validate( - { # mutable-ok: Pydantic validates this request-scoped translation session payload + { **(session or {}), "type": "translation", "model": requested_model_name, @@ -315,18 +309,16 @@ async def acreate_realtime_translation_client_secret( litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, - optional_params={ # mutable-ok: logging owns a mutable request metadata payload + optional_params={ "expires_after": expires_after, "session": session, }, - litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload - "api_base": resolved_api_base - }, + litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) request_data: Final = req.model_dump( exclude_none=True, - exclude={"model"}, # mutable-ok: Pydantic requires a mutable field-exclusion set + exclude={"model"}, ) request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) request_data["session"].pop("type", None) @@ -508,7 +500,7 @@ async def arealtime_translation_calls( litellm_params=litellm_params, ) session_config: Final = _with_resolved_session_model( - { # mutable-ok: provider routing requires an independently mutable session payload + { **(session or {}), "type": "translation", "model": model_name, @@ -518,13 +510,11 @@ async def arealtime_translation_calls( litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, - optional_params={ # mutable-ok: logging owns a mutable request metadata payload + optional_params={ "realtime_translation_calls": True, "session": session_config, }, - litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload - "api_base": resolved_api_base - }, + litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) return await base_llm_http_handler.async_realtime_calls_handler( diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 267e95d2773..8ee61c45cec 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -648,9 +648,10 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): @pytest.mark.asyncio -async def test_translation_session_update_omits_session_type(): +@pytest.mark.parametrize("beta_client", (False, True)) +async def test_translation_session_update_omits_session_type(beta_client: bool) -> None: client_ws = MagicMock() - client_ws.scope = {"headers": []} + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")] if beta_client else []} client_ws.receive_text = AsyncMock( side_effect=[ json.dumps( @@ -658,6 +659,8 @@ async def test_translation_session_update_omits_session_type(): "type": "session.update", "session": { "type": "translation", + "input_audio_format": "g711_ulaw", + "output_audio_format": "pcm16", "audio": {"output": {"language": "fr"}}, }, } @@ -681,6 +684,11 @@ async def test_translation_session_update_omits_session_type(): sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) assert "type" not in sent_to_backend["session"] assert sent_to_backend["session"]["audio"]["output"]["language"] == "fr" + assert sent_to_backend["session"]["audio"]["input"]["format"]["type"] == "audio/pcmu", ( + "OpenAI Realtime audio formats, checked 2026-10-06: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_audio_formats.py" + ) + assert sent_to_backend["session"]["audio"]["output"]["format"]["type"] == "audio/pcm" def test_translate_event_to_beta_renames_delta_types(): @@ -3068,8 +3076,22 @@ def test_translation_audio_duration_is_finalized_once(event_type: str): @pytest.mark.asyncio @pytest.mark.parametrize("output_bytes", (0, 48000)) +@pytest.mark.parametrize("format_event_type", ("session.created", "session.update")) +@pytest.mark.parametrize( + "audio_format,bytes_per_second", + ( + ("pcm16", 48000), + ("g711_ulaw", 8000), + ("g711_alaw", 8000), + ({"type": "audio/pcm", "rate": 16000}, 32000), + ({"type": "audio/pcmu", "rate": 8000}, 8000), + ({"type": "audio/pcma", "rate": 8000}, 8000), + ({"type": "audio/pcmu"}, 8000), + ({"type": "audio/pcma"}, 8000), + ), +) async def test_translation_disconnect_bills_sent_input_audio( - output_bytes: int, + output_bytes: int, format_event_type: str, audio_format: str | Mapping[str, object], bytes_per_second: int ) -> None: import base64 @@ -3082,8 +3104,21 @@ async def test_translation_disconnect_bills_sent_input_audio( model="gpt-realtime-translate", translation_session=True, ) + format_event: Final = { + "type": format_event_type, + "session": {"audio": {"input": {"format": audio_format}}}, + } + if format_event_type == "session.update": + await streaming._send_to_backend(json.dumps(format_event)) + else: + streaming._capture_translation_output_audio(format_event) await streaming._send_to_backend( - json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) + json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(bytes(bytes_per_second * 2)).decode(), + } + ) ) streaming._capture_translation_output_audio( {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index 704efe8323a..14e8bb34801 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,4 +1,7 @@ +import asyncio import json +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -9,6 +12,7 @@ from litellm.types.realtime import RealtimeQueryParams from openai import AsyncOpenAI, omit +from pydantic import TypeAdapter class DummySDKConnectionManager: @@ -487,24 +491,35 @@ def test_translation_url_uses_dedicated_path(): @pytest.mark.asyncio -async def test_translation_websocket_uses_direct_transport(): +@pytest.mark.parametrize("beta_client", (False, True)) +async def test_translation_websocket_uses_direct_transport(beta_client: bool) -> None: from litellm.llms.openai.realtime.handler import OpenAIRealtime - backend = AsyncMock() + backend: Final = AsyncMock() + pending: Final = asyncio.Event() - class TranslationConnectionManager: - async def __aenter__(self): - return backend + async def receive_backend() -> str: + await pending.wait() + return "" - async def __aexit__(self, exc_type, exc, tb): - return None - - websocket = MagicMock() - websocket.scope = {"headers": []} + backend.recv.side_effect = receive_backend + websocket: Final = MagicMock() + websocket.scope = {"headers": [(b"openai-beta", b"realtime=v1")] if beta_client else []} + websocket.receive_text = AsyncMock( + side_effect=( + json.dumps( + { + "type": "session.update", + "session": {"type": "translation", "audio": {"output": {"language": "fr"}}}, + } + ), + RuntimeError("connection closed"), + ) + ) websocket.close = AsyncMock() - logging_obj = MagicMock() - handler = OpenAIRealtime() - expected_url = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + logging_obj: Final = MagicMock() + handler: Final = OpenAIRealtime() + expected_url: Final = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" assert ( handler._construct_url( api_base="https://api.openai.com/v1", @@ -514,13 +529,7 @@ async def test_translation_websocket_uses_direct_transport(): == expected_url ) - with ( - patch("websockets.connect", return_value=TranslationConnectionManager()) as connect, - patch( # test-quality-ok: transport test replaces the unbounded streaming loop - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as streaming, - ): - streaming.return_value.bidirectional_forward = AsyncMock() + with patch("websockets.connect", return_value=DummySDKConnectionManager(backend)) as connect: await handler.async_realtime( model="gpt-realtime-translate", websocket=websocket, @@ -533,7 +542,9 @@ async def test_translation_websocket_uses_direct_transport(): connect.assert_called_once() assert connect.call_args.args[0] == expected_url - assert streaming.call_args.kwargs["translation_session"] is True + assert "OpenAI-Beta" not in connect.call_args.kwargs["additional_headers"] + sent: Final = TypeAdapter(Mapping[str, object]).validate_json(backend.send.call_args_list[0].args[0]) + assert sent["session"] == {"audio": {"output": {"language": "fr"}}} @pytest.mark.parametrize("sdk_client", [True, False]) diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index fb676ae9a61..14f57d95632 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -364,6 +364,10 @@ async def test_unknown_invocation_target_leaves_billing_unset(monkeypatch: pytes ("/v1beta/models/gemini-model:generateContent", "POST", True), ("/v1/realtime", "GET", True), ("/v1/realtime", "POST", False), + ("/v1/realtime/translations", "GET", True), + ("/v1/realtime/translations", "POST", False), + ("/v1/realtime/translations/client_secrets", "POST", False), + ("/v1/realtime/translations/calls", "POST", False), ("/v1/realtime/client_secrets", "POST", False), ("/mcp/tools/call", "POST", True), ("/a2a/target/message/send", "POST", True), @@ -437,7 +441,17 @@ def test_managed_inference_ignores_unsupported_query_model(): ) -@pytest.mark.parametrize("route", ["/realtime", "/v1/realtime", "/openai/v1/realtime"]) +@pytest.mark.parametrize( + "route", + ( + "/realtime", + "/v1/realtime", + "/openai/v1/realtime", + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", + ), +) def test_managed_realtime_requires_a_model_and_ignores_completion_defaults(route: str) -> None: from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request @@ -588,6 +602,7 @@ def test_registered_inference_routes_have_an_explicit_managed_access_decision(ro "/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/", "/interactions", "/agents", "/responses/{", "/responses/input_tokens", "/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions", + "/realtime/translations/client_secrets", "/realtime/translations/calls", )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models") concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model") assert managed_agent_route_allowed(concrete, None) is not unsupported, route From 5e28d359b9a6a9b4a5d902cfbea96b89e7a12ef0 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 10:02:36 -0500 Subject: [PATCH 22/45] fix(realtime): trust confirmed formats and authorize default models --- .../litellm_core_utils/realtime_streaming.py | 49 ++++--- .../auth/managed_authorization.py | 10 +- litellm/proxy/auth/user_api_key_auth.py | 1 + .../proxy/common_utils/http_parsing_utils.py | 10 ++ litellm/proxy/proxy_server.py | 17 +-- .../test_realtime_streaming.py | 120 ++++++++++++++++-- .../auth/test_managed_authorization.py | 75 ++++++++--- 7 files changed, 214 insertions(+), 68 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 04fed504038..24b678157ec 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -8,6 +8,7 @@ from enum import Enum, auto from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable +from openai.types.realtime.realtime_audio_formats import AudioPCM, AudioPCMA, AudioPCMU, RealtimeAudioFormats from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly @@ -207,23 +208,17 @@ def _gate_session_update( return {**client_session, "turn_detection": gate(None)} -_TRANSLATION_AUDIO_FORMAT: Final = TypeAdapter(str | Mapping[str, object] | None) +_TRANSLATION_AUDIO_FORMAT: Final = TypeAdapter(str | RealtimeAudioFormats | None) -def _translation_audio_bytes_per_second(audio_format: str | Mapping[str, object] | None) -> float | None: - if isinstance(audio_format, str): - return 8000.0 if audio_format in ("g711_ulaw", "g711_alaw") else None - if audio_format is None: - return None - format_type: Final = audio_format.get("type") - rate: Final = audio_format.get("rate") - if format_type in ("audio/pcmu", "audio/pcma") and rate is None: - return 8000.0 - if not isinstance(rate, (int, float)) or rate <= 0: - return None - if format_type == "audio/pcm": - return float(rate) * 2 - return float(rate) if format_type in ("audio/pcmu", "audio/pcma") else None +def _translation_audio_bytes_per_second(audio_format: str | RealtimeAudioFormats | None) -> float | None: + match audio_format: + case "pcm16" | AudioPCM(): + return 48000.0 + case "g711_ulaw" | "g711_alaw" | AudioPCMU() | AudioPCMA(): + return 8000.0 + case _: + return None class RealtimeEventNormalizer(Protocol): @@ -281,7 +276,7 @@ class RealTimeStreaming: self._is_translation_session = translation_session self._translation_input_seconds = 0.0 self._translation_input_bytes_per_second = 48000.0 - self._translation_output_audio_bytes = 0 + self._translation_output_seconds = 0.0 self._translation_output_bytes_per_second = 48000.0 self._translation_usage_finalized = False @@ -596,11 +591,7 @@ class RealTimeStreaming: if reported_input_seconds is not None else self._translation_input_seconds or None ) - synthetic_output_seconds: Final = ( - self._translation_output_audio_bytes / self._translation_output_bytes_per_second - if self._translation_output_audio_bytes > 0 - else None - ) + synthetic_output_seconds: Final = self._translation_output_seconds or None resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds if input_seconds is not None or resolved_output_seconds is not None: if self._should_store_message(event_obj): @@ -650,7 +641,7 @@ class RealTimeStreaming: decoded: Final = base64.b64decode(delta, validate=True) except (ValueError, TypeError): return - self._translation_output_audio_bytes += len(decoded) + self._translation_output_seconds += len(decoded) / self._translation_output_bytes_per_second def _capture_translation_input_audio(self, message: str) -> None: if not self._is_translation_session: @@ -659,7 +650,6 @@ class RealTimeStreaming: event: Final = _decode_json_object(message) except (json.JSONDecodeError, TypeError): return - self._capture_translation_audio_formats(event) if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str): return try: @@ -669,6 +659,8 @@ class RealTimeStreaming: self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second def _capture_translation_audio_formats(self, event_obj: Mapping[str, object]) -> None: + if event_obj.get("type") not in ("session.created", "session.updated"): + return session: Final = event_obj.get("session") if not isinstance(session, dict): return @@ -704,15 +696,18 @@ class RealTimeStreaming: ): self._translation_usage_finalized = True return - if self._translation_output_audio_bytes == 0 and self._translation_input_seconds == 0: + if self._translation_output_seconds == 0 and self._translation_input_seconds == 0: return - output_seconds: Final = self._translation_output_audio_bytes / self._translation_output_bytes_per_second synthetic_usage: Final = ( OpenAIRealtimeTranslationDurationUsage( - type="duration", input_seconds=self._translation_input_seconds, output_seconds=output_seconds + type="duration", + input_seconds=self._translation_input_seconds, + output_seconds=self._translation_output_seconds, ) if self._translation_input_seconds > 0 - else OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=self._translation_output_seconds + ) ) self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) self._translation_usage_finalized = True diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 229c5d23d74..3740d157249 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -115,13 +115,21 @@ def managed_inference_request( cli_model: str | None, path_model: object = None, query_model: object = None, + intent: str | None = None, ) -> dict[str, object]: from litellm.proxy.auth.route_checks import RouteChecks if route in _MANAGED_LIVE_ROUTES: return dict(body) if route in _MANAGED_REALTIME_ROUTES: - model: Final = query_model or body.get("model") + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + requested_model: Final = query_model if query_model is not None else body.get("model") + model: Final = ( + resolve_realtime_route_model(None, intent, route.endswith("/realtime/translations")) + if requested_model is None + else requested_model + ) if not isinstance(model, str) or not model: raise_identity_failure( AgentIdentityFailure(message="Managed inference requires an explicit or configured model") diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 27fa4731baf..e22d8058c0b 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -4065,6 +4065,7 @@ async def _authorize_authenticated_request( user_model, request.path_params.get("model") or request.path_params.get("model_name"), request.query_params.get("model"), + intent=request.query_params.get("intent"), ) if user_api_key_auth_obj.managed_agent_policy is not None else request_data diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 5d4dcda1e83..796ce3333f8 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -33,6 +33,16 @@ _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequ _DECODED_JSON: Final = TypeAdapter(object) +def resolve_realtime_route_model(model: str | None, intent: str | None, is_translation: bool) -> str | None: + if model is not None: + return model + if is_translation: + return "gpt-realtime-translate" + if intent == "transcription": + return "gpt-realtime-whisper" + return None + + def resolve_inference_model( body_model: object, settings: Mapping[str, object], diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d2c1d5c9268..997a73be66a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -502,6 +502,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF10 resolve_inference_model, safe_get_request_headers, ) +from litellm.proxy.common_utils.http_parsing_utils import ( + resolve_realtime_route_model as _resolve_realtime_route_model, +) from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( @@ -13655,20 +13658,6 @@ async def _reject_realtime_session( await _release_realtime_max_parallel_slot(user_api_key_dict) -def _resolve_realtime_route_model( - model: str | None, - intent: str | None, - is_translation: bool, -) -> str | None: - if model is not None: - return model - if is_translation: - return "gpt-realtime-translate" - if intent == "transcription": - return "gpt-realtime-whisper" - return None - - def _resolve_realtime_upstream_query_model( model: str | None, intent: str | None, diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 8ee61c45cec..f3e72050d4a 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3076,14 +3076,14 @@ def test_translation_audio_duration_is_finalized_once(event_type: str): @pytest.mark.asyncio @pytest.mark.parametrize("output_bytes", (0, 48000)) -@pytest.mark.parametrize("format_event_type", ("session.created", "session.update")) +@pytest.mark.parametrize("format_event_type", ("session.created", "session.updated")) @pytest.mark.parametrize( "audio_format,bytes_per_second", ( ("pcm16", 48000), ("g711_ulaw", 8000), ("g711_alaw", 8000), - ({"type": "audio/pcm", "rate": 16000}, 32000), + ({"type": "audio/pcm", "rate": 24000}, 48000), ({"type": "audio/pcmu", "rate": 8000}, 8000), ({"type": "audio/pcma", "rate": 8000}, 8000), ({"type": "audio/pcmu"}, 8000), @@ -3108,10 +3108,7 @@ async def test_translation_disconnect_bills_sent_input_audio( "type": format_event_type, "session": {"audio": {"input": {"format": audio_format}}}, } - if format_event_type == "session.update": - await streaming._send_to_backend(json.dumps(format_event)) - else: - streaming._capture_translation_output_audio(format_event) + streaming._capture_translation_output_audio(format_event) await streaming._send_to_backend( json.dumps( { @@ -3134,6 +3131,96 @@ async def test_translation_disconnect_bills_sent_input_audio( ] +@pytest.mark.asyncio +@pytest.mark.parametrize("rate", (24000, 1, 0, -1, float("nan"), float("inf"))) +async def test_translation_client_format_updates_do_not_change_billed_duration(rate: int | float) -> None: + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"input": {"format": "g711_ulaw"}, "output": {"format": "g711_ulaw"}}}, + } + ) + input_event: Final = json.dumps( + {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(8000)).decode()} + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(8000)).decode()} + await streaming._send_to_backend(input_event) + streaming._capture_translation_output_audio(output_event) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "audio": { + "input": {"format": {"type": "audio/pcm", "rate": rate}}, + "output": {"format": {"type": "audio/pcm", "rate": rate}}, + } + }, + } + ) + ) + await streaming._send_to_backend(input_event) + streaming._capture_translation_output_audio(output_event) + streaming._finalize_translation_usage() + + assert streaming.messages == [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 2.0}} + ] + + +@pytest.mark.asyncio +async def test_translation_confirmed_format_changes_preserve_previous_audio_duration() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(48000)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + { + "type": "session.updated", + "session": {"audio": {"input": {"format": "g711_alaw"}, "output": {"format": "g711_alaw"}}}, + } + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(8000)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(8000)).decode()} + ) + streaming._finalize_translation_usage() + + assert streaming.messages == [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 2.0}} + ] + + +@pytest.mark.parametrize("rate", (1, 0, -1, float("nan"), float("inf"), 10**400)) +def test_translation_invalid_provider_format_preserves_audio_duration(rate: int | float) -> None: + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=MagicMock(), logging_obj=MagicMock(), translation_session=True + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + streaming._capture_translation_output_audio(output_event) + streaming._capture_translation_output_audio( + {"type": "session.updated", "session": {"audio": {"output": {"format": {"type": "audio/pcm", "rate": rate}}}}} + ) + streaming._capture_translation_output_audio(output_event) + streaming._finalize_translation_usage() + + assert streaming.messages == [{"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}}] + + @pytest.mark.asyncio async def test_translation_failed_audio_send_is_not_billed() -> None: backend: Final = MagicMock() @@ -3218,7 +3305,9 @@ def test_translation_does_not_duplicate_provider_duration_usage(): model="gpt-realtime-translate", translation_session=True, ) - streaming._translation_output_audio_bytes = 48000 + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) streaming.messages.append({"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}}) streaming._finalize_translation_usage() @@ -3266,7 +3355,9 @@ def test_translation_preserves_input_only_provider_usage( model="gpt-realtime-translate", translation_session=True, ) - streaming._translation_output_audio_bytes = output_audio_bytes + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_audio_bytes)).decode()} + ) streaming._capture_translation_output_audio( {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} ) @@ -3286,7 +3377,9 @@ def test_translation_retained_input_only_close_event_bills_captured_output(monke model="gpt-realtime-translate", translation_session=True, ) - streaming._translation_output_audio_bytes = 48000 + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) close_event: Final = {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} streaming._capture_translation_output_audio(close_event) @@ -3308,10 +3401,15 @@ def test_translation_malformed_provider_duration_uses_captured_audio(malformed_o model="gpt-realtime-translate", translation_session=True, ) - streaming._translation_output_audio_bytes = 48000 + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) streaming._capture_translation_output_audio( - {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": malformed_output}} + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": malformed_output}, + } ) streaming._finalize_translation_usage() diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index 14f57d95632..8513a59cfe2 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -447,9 +447,6 @@ def test_managed_inference_ignores_unsupported_query_model(): "/realtime", "/v1/realtime", "/openai/v1/realtime", - "/realtime/translations", - "/v1/realtime/translations", - "/openai/v1/realtime/translations", ), ) def test_managed_realtime_requires_a_model_and_ignores_completion_defaults(route: str) -> None: @@ -479,6 +476,35 @@ def test_managed_live_upgrade_defers_the_model_to_the_session_start_frame(route: ) == {"model": "frame-model"} +@pytest.mark.parametrize( + "route", ("/realtime/translations", "/v1/realtime/translations", "/openai/v1/realtime/translations") +) +@pytest.mark.parametrize("query_model", (None, "requested")) +def test_managed_translation_authorizes_the_same_model_as_the_endpoint(route: str, query_model: str | None) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + admitted: Final = managed_inference_request( + route, {}, {"completion_model": "unrelated-default"}, "cli", query_model=query_model + ) + + assert admitted["model"] == resolve_realtime_route_model(query_model, None, is_translation=True) + assert isinstance(admitted["model"], str) and admitted["model"] + + +@pytest.mark.parametrize("route", ("/realtime", "/v1/realtime", "/openai/v1/realtime")) +def test_managed_transcription_authorizes_the_same_model_as_the_endpoint(route: str) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + admitted: Final = managed_inference_request( + route, {}, {"completion_model": "unrelated-default"}, "cli", intent="transcription" + ) + + assert admitted["model"] == resolve_realtime_route_model(None, "transcription", is_translation=False) + assert isinstance(admitted["model"], str) and admitted["model"] + + @pytest.mark.parametrize("mode,user", [("autonomous", None), ("delegated", "verified-human")]) def test_matching_identity_revision_and_execution_mode_pass_admission(mode: str, user: str | None) -> None: context: Final = ManagedAgentContext.model_validate( @@ -587,22 +613,41 @@ async def test_ordinary_agent_admission_preserves_legacy_authentication( @pytest.mark.parametrize( "route", - tuple(dict.fromkeys( - LiteLLMRoutes.openai_routes.value - + LiteLLMRoutes.anthropic_routes.value - + LiteLLMRoutes.google_routes.value - )), + tuple( + dict.fromkeys( + LiteLLMRoutes.openai_routes.value + LiteLLMRoutes.anthropic_routes.value + LiteLLMRoutes.google_routes.value + ) + ), ) def test_registered_inference_routes_have_an_explicit_managed_access_decision(route: str) -> None: from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_route_allowed normalized: Final = route.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") - unsupported: Final = normalized.startswith(( - "/videos", "/batches", "/files", "/fine_tuning", "/assistants", "/threads", "/utils/", - "/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/", - "/interactions", "/agents", "/responses/{", "/responses/input_tokens", - "/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions", - "/realtime/translations/client_secrets", "/realtime/translations/calls", - )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models") + unsupported: Final = normalized.startswith( + ( + "/videos", + "/batches", + "/files", + "/fine_tuning", + "/assistants", + "/threads", + "/utils/", + "/vector_stores", + "/vector_store/", + "/search", + "/containers", + "/skills", + "/claude-code/", + "/interactions", + "/agents", + "/responses/{", + "/responses/input_tokens", + "/realtime/client_secrets", + "/realtime/calls", + "/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/realtime/translations/calls", + ) + ) or normalized in ("/models", "/cursor/models", "/cursor/v1/models") concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model") assert managed_agent_route_allowed(concrete, None) is not unsupported, route From 220fe868fc470317ed4666cd4b205f812471e056 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 10:27:56 -0500 Subject: [PATCH 23/45] fix(realtime): preserve absent models and reject unsupported guardrails --- .../litellm_core_utils/realtime_streaming.py | 13 +++++- litellm/proxy/common_utils/realtime_utils.py | 3 +- .../test_realtime_streaming.py | 44 +++++++++++++++++++ .../proxy/common_utils/test_realtime_cache.py | 25 ++++++++++- 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 24b678157ec..cef6a24aba1 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -218,7 +218,8 @@ def _translation_audio_bytes_per_second(audio_format: str | RealtimeAudioFormats case "g711_ulaw" | "g711_alaw" | AudioPCMU() | AudioPCMA(): return 8000.0 case _: - return None + pass + return None class RealtimeEventNormalizer(Protocol): @@ -1954,6 +1955,16 @@ class RealTimeStreaming: return ClientLoopExit.CLIENT_DISCONNECTED async def bidirectional_forward(self) -> None: + if self._is_translation_session and self._has_realtime_guardrails(): + await self.websocket.send_text( + realtime_error_event( + "Translation sessions cannot enforce configured realtime guardrails", error_type="guardrail_error" + ) + ) + await self.websocket.close( + code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" + ) + return forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages()) client_task: Final = asyncio.create_task(self.client_ack_messages()) try: diff --git a/litellm/proxy/common_utils/realtime_utils.py b/litellm/proxy/common_utils/realtime_utils.py index c641748843e..d4fd4556ed4 100644 --- a/litellm/proxy/common_utils/realtime_utils.py +++ b/litellm/proxy/common_utils/realtime_utils.py @@ -1,3 +1,4 @@ +import json from functools import lru_cache from typing import Final @@ -10,7 +11,7 @@ def realtime_request_body(model: str | None) -> bytes: Generate the realtime websocket request body. Cached with LRU semantics to avoid repeated string formatting work while keeping memory usage bounded. """ - return f'{{"model": "{model or ""}"}}'.encode() + return json.dumps({"model": model}).encode() _realtime_request_body: Final = realtime_request_body diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index f3e72050d4a..b896da11c3c 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3236,6 +3236,50 @@ async def test_translation_failed_audio_send_is_not_billed() -> None: assert streaming.messages == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("default_on,requested", ((False, False), (True, False), (False, True))) +@pytest.mark.parametrize( + "event_hook", + (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call), +) +async def test_translation_rejects_applicable_guardrails_before_forwarding_audio( + monkeypatch: pytest.MonkeyPatch, default_on: bool, requested: bool, event_hook: GuardrailEventHooks +) -> None: + guardrail: Final = CustomGuardrail(guardrail_name="speech-policy", event_hook=event_hook, default_on=default_on) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + input_event: Final = json.dumps( + {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(4800)).decode()} + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(4800)).decode()} + client: Final = _ga_client_ws() + client.receive_text = AsyncMock(side_effect=[input_event, ConnectionClosed(None, None)]) + client.send_text = AsyncMock() + client.close = AsyncMock() + backend: Final = MagicMock() + backend.send = AsyncMock() + backend.recv = AsyncMock(side_effect=[json.dumps(output_event), ConnectionClosed(None, None)]) + streaming: Final = RealTimeStreaming( + websocket=client, + backend_ws=backend, + logging_obj=MagicMock(), + translation_session=True, + request_data={"litellm_metadata": {"guardrails": ["speech-policy"] if requested else []}}, + ) + + await streaming.bidirectional_forward() + + messages: Final = tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) + if default_on or requested: + assert messages[0]["error"]["type"] == "guardrail_error" + client.close.assert_awaited_once_with( + code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" + ) + backend.send.assert_not_awaited() + else: + assert output_event in messages + backend.send.assert_awaited_once_with(input_event) + + @pytest.mark.asyncio @pytest.mark.parametrize("retain_close", (False, True)) @pytest.mark.parametrize("reported_input,expected_input", [(None, 2.0), (0.0, 0.0), (0.25, 0.25)]) diff --git a/tests/unit/proxy/common_utils/test_realtime_cache.py b/tests/unit/proxy/common_utils/test_realtime_cache.py index 6f5751fa923..9750a17ae14 100644 --- a/tests/unit/proxy/common_utils/test_realtime_cache.py +++ b/tests/unit/proxy/common_utils/test_realtime_cache.py @@ -1,6 +1,8 @@ -from typing import Any, cast +from collections.abc import Mapping +from typing import Any, Final, cast import pytest +from pydantic import TypeAdapter from litellm.proxy.common_utils.realtime_utils import realtime_request_body from litellm.proxy.proxy_server import _realtime_query_params_template @@ -40,6 +42,27 @@ def test_realtime_request_body_caches_each_model_separately(): assert gpt4o_body_first is not gpt4o_mini_body +@pytest.mark.parametrize("is_translation", (False, True)) +def test_realtime_request_body_preserves_default_model_authorization(is_translation: bool) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + body: Final = TypeAdapter(Mapping[str, object]).validate_json(_realtime_request_body(None)) + route: Final = "/v1/realtime/translations" if is_translation else "/v1/realtime" + intent: Final = None if is_translation else "transcription" + admitted: Final = managed_inference_request(route, body, {}, None, intent=intent) + + assert admitted["model"] == resolve_realtime_route_model(None, intent, is_translation) + assert isinstance(admitted["model"], str) and admitted["model"] + + +@pytest.mark.parametrize("model", (None, "requested", 'model"with\\quotes', "model\nwith-newline")) +def test_realtime_request_body_preserves_model_names(model: str | None) -> None: + body: Final = TypeAdapter(Mapping[str, object]).validate_json(_realtime_request_body(model)) + + assert body["model"] == model + + def test_realtime_query_params_template_caches_each_pair_separately(): params_with_intent_first = _realtime_query_params_template("gpt-4o", "intent-a") params_with_intent_second = _realtime_query_params_template("gpt-4o", "intent-a") From e6f9403434e057fb184b80c776450da71841e29c Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 10:55:29 -0500 Subject: [PATCH 24/45] fix(realtime): reject translation policy pipelines --- .../litellm_core_utils/realtime_streaming.py | 15 +++- litellm/realtime_api/main.py | 14 ++-- .../test_realtime_streaming.py | 79 +++++++++++++++++++ 3 files changed, 98 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index cef6a24aba1..5b55a5223bf 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -255,7 +255,7 @@ class RealTimeStreaming: provider_config: BaseRealtimeConfig | None = None, model: str = "", user_api_key_dict: object | None = None, - request_data: dict | None = None, + request_data: Mapping[str, object] | None = None, backend_uses_beta_protocol: bool | None = None, force_transcription_model: str | None = None, event_normalizer: RealtimeEventNormalizer | None = None, @@ -302,7 +302,7 @@ class RealTimeStreaming: self.current_delta_type: ALL_DELTA_TYPES | None = None self.session_configuration_request: str | None = None self.user_api_key_dict = user_api_key_dict - self.request_data: dict = request_data or {} + self.request_data: Mapping[str, object] = request_data or {} # Violation counter for end_session_after_n_fails support self._violation_count: int = 0 # When a text message is blocked, hold the guardrail reason so the next @@ -1176,9 +1176,18 @@ class RealTimeStreaming: ) def _has_realtime_guardrails(self) -> bool: - """Return True if any callback is registered for realtime guardrail event types.""" from litellm.types.guardrails import GuardrailEventHooks + if any( + isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket + for bucket in (self.request_data.get("metadata"), self.request_data.get("litellm_metadata")) + ): + from litellm.proxy.utils import pipeline_managed_guardrail_names + + if pipeline_managed_guardrail_names(self.request_data, "pre_call") or pipeline_managed_guardrail_names( + self.request_data, "post_call" + ): + return True return self._has_realtime_guardrails_for_event_hooks( [ GuardrailEventHooks.realtime_input_transcription, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index de807867bed..2c946a3dd91 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast from typing_extensions import ReadOnly, TypedDict, assert_never import httpx +from pydantic import TypeAdapter import litellm from litellm.constants import ( @@ -66,6 +67,7 @@ vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() _EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) +_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]: @@ -104,13 +106,11 @@ def _with_resolved_session_model(session: dict[str, object], model_name: str) -> return {**session, "model": model_name} -def _build_litellm_metadata(kwargs: dict) -> dict[str, object]: - """Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider).""" - metadata: Final[dict[str, object]] = {**(kwargs.get("litellm_metadata") or {})} - guardrails: Final = (kwargs.get("metadata") or {}).get("guardrails") or kwargs.get("guardrails") or [] - if guardrails: - metadata["guardrails"] = guardrails - return metadata +def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]: + request_metadata: Final = _METADATA_ADAPTER.validate_python(kwargs.get("metadata") or {}) + internal_metadata: Final = _METADATA_ADAPTER.validate_python(kwargs.get("litellm_metadata") or {}) + guardrails: Final = request_metadata.get("guardrails") or kwargs.get("guardrails") + return {**request_metadata, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})} def _get_realtime_http_provider_config( diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index b896da11c3c..d9fab04b7ae 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3,6 +3,7 @@ import base64 import json from collections.abc import Coroutine, Mapping, Sequence from dataclasses import dataclass +from datetime import UTC, datetime from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -19,6 +20,7 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.realtime_streaming import ( RealTimeStreaming, client_sent_openai_beta_realtime_header, @@ -29,10 +31,14 @@ from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.litellm_pre_call_utils import _apply_resolved_guardrails_to_metadata +from litellm.realtime_api.main import _build_litellm_metadata from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeResponseTypedDict from litellm.types.utils import GenericGuardrailAPIInputs +from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyMatchContext +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep def _mock_logging() -> MagicMock: @@ -3280,6 +3286,79 @@ async def test_translation_rejects_applicable_guardrails_before_forwarding_audio backend.send.assert_awaited_once_with(input_event) +@pytest.mark.asyncio +@pytest.mark.parametrize("metadata_bucket", ("metadata", "litellm_metadata")) +@pytest.mark.parametrize("mode", ("pre_call", "post_call")) +@pytest.mark.parametrize("selected,condition_matches", ((True, True), (True, False), (False, True))) +async def test_translation_rejects_selected_pipeline_guardrails_before_forwarding_audio( + monkeypatch: pytest.MonkeyPatch, + metadata_bucket: str, + mode: Literal["pre_call", "post_call"], + selected: bool, + condition_matches: bool, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="speech-policy", event_hook=GuardrailEventHooks(mode), default_on=False + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + request_data: Final[dict[str, object]] = {"metadata": {}, "litellm_metadata": {"user_id": "qa-user"}} + _apply_resolved_guardrails_to_metadata( + data=request_data, + metadata_variable_name=metadata_bucket, + context=PolicyMatchContext(model="qa-translation"), + policy_names=["translation-policy"] if selected else [], + policies={ + "translation-policy": Policy( + condition=PolicyCondition(model="qa-translation" if condition_matches else "unrelated-model"), + pipeline=GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="speech-policy", on_fail="block")]), + ) + }, + ) + input_event: Final = json.dumps( + {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(4800)).decode()} + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(4800)).decode()} + client: Final = _ga_client_ws() + client.receive_text = AsyncMock(side_effect=[input_event, ConnectionClosed(None, None)]) + client.send_text = AsyncMock() + client.close = AsyncMock() + backend: Final = MagicMock() + backend.send = AsyncMock() + backend.recv = AsyncMock(side_effect=[json.dumps(output_event), ConnectionClosed(None, None)]) + logging_obj: Final = Logging( + model="gpt-realtime-translate", + messages=[], + stream=True, + call_type="_arealtime", + start_time=datetime(2026, 1, 1, tzinfo=UTC), + litellm_call_id="pipeline-rejection", + function_id="pipeline-rejection", + ) + worker: Final = _InlineLoggingWorker() + streaming: Final = RealTimeStreaming( + websocket=client, + backend_ws=backend, + logging_obj=logging_obj, + translation_session=True, + request_data={"litellm_metadata": _build_litellm_metadata(request_data)}, + logging_worker=worker, + ) + + await streaming.bidirectional_forward() + await worker.drain() + + messages: Final = tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) + if selected and condition_matches: + assert messages[0]["error"]["type"] == "guardrail_error" + client.close.assert_awaited_once_with( + code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" + ) + backend.send.assert_not_awaited() + else: + assert output_event in messages + backend.send.assert_awaited_once_with(input_event) + + @pytest.mark.asyncio @pytest.mark.parametrize("retain_close", (False, True)) @pytest.mark.parametrize("reported_input,expected_input", [(None, 2.0), (0.0, 0.0), (0.25, 0.25)]) From 318f0a61de8cb93e231b010669a6ec1767c02d32 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 11:20:19 -0500 Subject: [PATCH 25/45] chore(ci): rerun transient infrastructure timing failure From a5d620606f4b003b190055351870003e09b0d0b8 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Fri, 16 Oct 2026 10:48:41 -0500 Subject: [PATCH 26/45] fix(realtime): reconcile audio protocols with current upstream --- litellm/__init__.py | 4 +- litellm/cost_calculator.py | 55 +- .../litellm_core_utils/realtime_streaming.py | 114 +- litellm/llms/azure/audio_transcriptions.py | 17 +- .../audio_transcription/transformation.py | 18 +- litellm/llms/custom_httpx/llm_http_handler.py | 9 +- litellm/llms/openai/realtime/handler.py | 1 - litellm/llms/openai/transcriptions/handler.py | 22 +- ...odel_prices_and_context_window_backup.json | 109 +- litellm/proxy/_lazy_openapi_snapshot.json | 1816 ++++++++++++++++- litellm/proxy/proxy_server.py | 2 + litellm/proxy/realtime_endpoints/endpoints.py | 26 + litellm/realtime_api/main.py | 14 +- litellm/types/realtime.py | 30 +- litellm/types/utils.py | 12 +- model_prices_and_context_window.json | 109 +- scripts/check_openapi_docs.py | 8 +- .../test_realtime_streaming.py | 28 +- .../realtime/test_openai_realtime_handler.py | 4 +- .../proxy/common_utils/test_realtime_cache.py | 4 +- tests/unit/test_check_openapi_docs.py | 15 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 410 +++- 22 files changed, 2485 insertions(+), 342 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index a1de9f90f5e..58cbc4d8769 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1512,9 +1512,9 @@ from .realtime_api.main import ( _arealtime, acreate_realtime_client_secret, acreate_realtime_transcription_session, - acreate_realtime_translation_client_secret, + acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret, arealtime_calls, - arealtime_translation_calls, + arealtime_translation_calls as arealtime_translation_calls, ) from .responses.main import _aresponses_websocket diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9ce1be1a41a..296b8ad2ae6 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -22,11 +22,11 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR -from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -324,7 +324,7 @@ def _transcription_uses_token_pricing( if usage_block is None: return False - model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) has_token_pricing: Final = any( model_info.get(field) for field in ( @@ -1247,19 +1247,20 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage: prompt_tokens_details=_decisions_prompt_tokens_details(usage), ) + def _get_transcription_usage_duration(completion_response: object) -> float | None: + response_fields: Final = as_str_mapping(completion_response) usage_object: Final = ( - completion_response.get("usage") - if isinstance(completion_response, dict) - else getattr(completion_response, "usage", None) + response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage") ) + usage_fields: Final = as_str_mapping(usage_object) usage_type: Final = ( - usage_object.get("type") if isinstance(usage_object, dict) else getattr(usage_object, "type", None) + usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type") ) if usage_type != "duration": return None seconds: Final = ( - usage_object.get("seconds") if isinstance(usage_object, dict) else getattr(usage_object, "seconds", None) + usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds") ) return normalized_audio_duration_seconds(seconds) @@ -3615,7 +3616,13 @@ def handle_realtime_stream_cost_calculation( input_cost: Final = input_cost_per_token + live_response_input_cost output_cost: Final = output_cost_per_token + live_response_output_cost total_cost: Final = ( - input_cost + output_cost + transcription_cost + translation_cost + live_voice_cost + audio_duration_cost + text_message_cost + input_cost + + output_cost + + transcription_cost + + translation_cost + + live_voice_cost + + audio_duration_cost + + text_message_cost ) additional_costs: Final = tuple( (cost_name, cost) @@ -3651,8 +3658,9 @@ def handle_realtime_translation_cost_calculation( usage_events: Final = tuple( usage for result in results - if result.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE - if isinstance(usage := result.get("usage"), dict) + if (event := as_str_mapping(result)) is not None + and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE + and (usage := as_str_mapping(event.get("usage"))) is not None ) input_seconds: Final = sum( seconds @@ -3660,9 +3668,9 @@ def handle_realtime_translation_cost_calculation( if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None ) output_seconds: Final = sum( - seconds + output_duration for usage in usage_events - if (seconds := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None + if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None ) if input_seconds <= 0 and output_seconds <= 0: return 0.0 @@ -3681,9 +3689,9 @@ def handle_realtime_translation_cost_calculation( ) output_cost_per_second: Final = next( ( - rate + output_rate for info in model_infos - if (rate := _declared_transcription_rate(info, ("output_cost_per_second",))) is not None + if (output_rate := _declared_transcription_rate(info, ("output_cost_per_second",))) is not None ), 0.0, ) @@ -3705,9 +3713,11 @@ def handle_realtime_transcription_cost_calculation( - {"type": "duration", "seconds": } → priced via input_cost_per_second - {"type": "tokens", "input_tokens": ...} → priced via input/audio token cost """ - completed_events: Final = [ - cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE - ] + completed_events: Final = tuple( + event + for result in results + if (event := as_str_mapping(result)) is not None and event.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE + ) if not completed_events: return 0.0 @@ -3717,11 +3727,10 @@ def handle_realtime_transcription_cost_calculation( _get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None ) - total_cost = 0.0 - for event in completed_events: - usage = event.get("usage") or {} - total_cost += _transcription_usage_cost(usage, model_info, override_info) - return total_cost + return sum( + _transcription_usage_cost(as_str_mapping(event.get("usage")) or {}, model_info, override_info) + for event in completed_events + ) def _get_transcription_model_name_from_results( diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 5b55a5223bf..e5b30c2a0e4 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -22,6 +22,7 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( @@ -208,7 +209,9 @@ def _gate_session_update( return {**client_session, "turn_detection": gate(None)} -_TRANSLATION_AUDIO_FORMAT: Final = TypeAdapter(str | RealtimeAudioFormats | None) +_TRANSLATION_AUDIO_FORMAT: Final[TypeAdapter[str | RealtimeAudioFormats | None]] = TypeAdapter( + str | RealtimeAudioFormats | None +) def _translation_audio_bytes_per_second(audio_format: str | RealtimeAudioFormats | None) -> float | None: @@ -385,12 +388,12 @@ class RealTimeStreaming: def _should_store_message( self, - message_obj: dict | OpenAIRealtimeEvents, + message_obj: Mapping[str, object], ) -> bool: _msg_type: Final = message_obj["type"] if "type" in message_obj else None if self.logged_real_time_event_types == "*": return True - if _msg_type and _msg_type in self.logged_real_time_event_types: + if isinstance(_msg_type, str) and _msg_type in self.logged_real_time_event_types: return True return False @@ -491,7 +494,7 @@ class RealTimeStreaming: input_messages, so it is not duplicated into the response log here. """ try: - usage: Final = event_obj.get("usage") + usage: Final = as_str_mapping(event_obj.get("usage")) if usage is None: return # If this event type is already captured by store_message (e.g. the user @@ -580,12 +583,12 @@ class RealTimeStreaming: if not self._is_translation_session: return if event_obj.get("type") == "session.closed": - usage: Final = event_obj.get("usage") + usage: Final = as_str_mapping(event_obj.get("usage")) output_seconds: Final = ( - normalized_audio_duration_seconds(usage.get("output_seconds")) if isinstance(usage, dict) else None + normalized_audio_duration_seconds(usage.get("output_seconds")) if usage is not None else None ) reported_input_seconds: Final = ( - normalized_audio_duration_seconds(usage.get("input_seconds")) if isinstance(usage, dict) else None + normalized_audio_duration_seconds(usage.get("input_seconds")) if usage is not None else None ) input_seconds: Final = ( reported_input_seconds @@ -662,14 +665,14 @@ class RealTimeStreaming: def _capture_translation_audio_formats(self, event_obj: Mapping[str, object]) -> None: if event_obj.get("type") not in ("session.created", "session.updated"): return - session: Final = event_obj.get("session") - if not isinstance(session, dict): + session: Final = as_str_mapping(event_obj.get("session")) + if session is None: return - audio: Final = session.get("audio") - if not isinstance(audio, dict): + audio: Final = as_str_mapping(session.get("audio")) + if audio is None: return - for direction, configuration in (("input", audio.get("input")), ("output", audio.get("output"))): - if not isinstance(configuration, dict): + for direction, raw_configuration in (("input", audio.get("input")), ("output", audio.get("output"))): + if (configuration := as_str_mapping(raw_configuration)) is None: continue try: if ( @@ -797,17 +800,17 @@ class RealTimeStreaming: self, session: Mapping[str, object], ) -> tuple[str, ...]: - audio: Final = session.get("audio") - audio_input: Final = audio.get("input") if isinstance(audio, dict) else None - nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + nested_transcription: Final = audio_input.get("transcription") if audio_input is not None else None nested_model: Final = self._transcription_model_value(nested_transcription) flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription")) return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None)) def _transcription_model_value(self, transcription_config: object) -> str | None: - if not isinstance(transcription_config, dict): + if (fields := as_str_mapping(transcription_config)) is None: return None - model: Final = transcription_config.get("model") + model: Final = fields.get("model") if isinstance(model, str) and model: return model return None @@ -816,47 +819,40 @@ class RealTimeStreaming: message_obj: Final = self._session_update_message_obj(message) if message_obj is None: return message - session: Final = message_obj.get("session") - if not isinstance(session, dict): + session: Final = as_str_mapping(message_obj.get("session")) + if session is None: return message - - transcription: Final = session.get("input_audio_transcription") - rewrite_flat: Final = isinstance(transcription, dict) and transcription.get("model") != authorized_model - if isinstance(transcription, dict) and rewrite_flat: - session["input_audio_transcription"] = { - **transcription, - "model": authorized_model, - } - - audio: Final = session.get("audio") - audio_input: Final = audio.get("input") if isinstance(audio, dict) else None - nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None - rewrite_nested: Final = ( - isinstance(audio, dict) - and isinstance(audio_input, dict) - and isinstance(nested_transcription, dict) - and nested_transcription.get("model") != authorized_model + transcription: Final = as_str_mapping(session.get("input_audio_transcription")) + rewrite_flat: Final = transcription is not None and transcription.get("model") != authorized_model + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + nested_transcription: Final = ( + as_str_mapping(audio_input.get("transcription")) if audio_input is not None else None + ) + rewrite_nested: Final = ( + nested_transcription is not None and nested_transcription.get("model") != authorized_model ) - if ( - isinstance(audio, dict) - and isinstance(audio_input, dict) - and isinstance(nested_transcription, dict) - and rewrite_nested - ): - session["audio"] = { - **audio, - "input": { - **audio_input, - "transcription": { - **nested_transcription, - "model": authorized_model, - }, - }, - } - if not rewrite_flat and not rewrite_nested: return message - return json.dumps(message_obj) + updated_session: Final = { + **session, + **( + {"input_audio_transcription": {**transcription, "model": authorized_model}} + if transcription is not None and rewrite_flat + else {} + ), + **( + { + "audio": { + **audio, + "input": {**audio_input, "transcription": {**nested_transcription, "model": authorized_model}}, + } + } + if audio is not None and audio_input is not None and nested_transcription is not None and rewrite_nested + else {} + ), + } + return json.dumps({**message_obj, "session": updated_session}) def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -878,8 +874,8 @@ class RealTimeStreaming: message_obj: Final = self._session_update_message_obj(message) if message_obj is None: return message - session: Final = message_obj.get("session") - if isinstance(session, dict) and session.get("type") == "transcription": + session: Final = as_str_mapping(message_obj.get("session")) + if session is not None and session.get("type") == "transcription": self._is_transcription_session = True return self._rewrite_session_update_transcription_model(message, self._force_transcription_model) @@ -890,8 +886,8 @@ class RealTimeStreaming: message_obj: Final = self._session_update_message_obj(message) if message_obj is None: return message - session: Final = message_obj.get("session") - if not isinstance(session, dict): + session: Final = as_str_mapping(message_obj.get("session")) + if session is None: return message nested_models: Final = self._nested_transcription_models_from_session(session) if not nested_models: diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 9e3d5b27b6c..3600ac9e5e1 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,4 +1,4 @@ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Any, Final from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI @@ -7,6 +7,7 @@ from pydantic import BaseModel import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data from litellm.types.utils import FileTypes from litellm.utils import ( @@ -43,8 +44,10 @@ class AzureAudioTranscription(AzureChatCompletion): ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) - model_info: Final = litellm.model_cost.get(f"azure/{model}") - provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + model_info: Final = as_str_mapping(litellm.model_cost.get(f"azure/{model}")) + provider_specific_entry: Final = ( + as_str_mapping(model_info.get("provider_specific_entry")) if model_info is not None else None + ) requires_deployment_api: Final = model_info is None or ( provider_specific_entry is not None and provider_specific_entry.get("transcription_deployment_api") == 1 ) @@ -98,7 +101,7 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - response: Final = azure_client.audio.transcriptions.create( + response: Final = azure_client.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) @@ -131,7 +134,7 @@ class AzureAudioTranscription(AzureChatCompletion): self, audio_file: FileTypes, model: str, - data: dict, + data: Mapping[str, object], model_response: TranscriptionResponse, timeout: float, logging_obj: "LiteLLMLoggingObj", @@ -173,12 +176,12 @@ class AzureAudioTranscription(AzureChatCompletion): ) if data.get("stream") is True: - return await async_azure_client.audio.transcriptions.create( + return await async_azure_client.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) - raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( + raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 2078b063453..5d4efd32491 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,9 +1,12 @@ from abc import ABC, abstractmethod +from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -11,18 +14,21 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse +_TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object]) -def sdk_compatible_transcription_request_data(data: dict) -> dict: + +def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]: + request_data: Final = _TRANSCRIPTION_REQUEST.validate_python(data) extension_keys: Final = ("keywords", "languages") - extension_body: Final = {key: data[key] for key in extension_keys if key in data} + extension_body: Final = {key: request_data[key] for key in extension_keys if key in request_data} if not extension_body: - return data + return request_data - existing_extra_body: Final = data.get("extra_body") + existing_extra_body: Final = request_data.get("extra_body") return { - **{key: value for key, value in data.items() if key not in extension_keys}, + **{key: value for key, value in request_data.items() if key not in extension_keys}, "extra_body": { - **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), + **(as_str_mapping(existing_extra_body) or {}), **extension_body, }, } diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1365b9d30ab..46b1901895a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -29,11 +29,10 @@ from httpx import USE_CLIENT_DEFAULT from httpx._types import FileContent from openai import AsyncOpenAI from openai.types.file_deleted import FileDeleted +from openai.types.realtime import RealtimeSessionCreateRequestParam from pydantic import TypeAdapter from typing_extensions import ReadOnly -from openai.types.realtime import RealtimeSessionCreateRequestParam - import litellm import litellm.litellm_core_utils import litellm.types @@ -7124,6 +7123,12 @@ class BaseLLMHTTPHandler: normalized_api_base: Final = trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" if isinstance(client, AsyncOpenAI): return client, False, normalized_api_base + if isinstance(client, AsyncHTTPHandler): + return ( + AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0, http_client=client.client), + False, + normalized_api_base, + ) return AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0), True, normalized_api_base @staticmethod diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 259a920b975..c39cb74f224 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -302,7 +302,6 @@ class OpenAIRealtime(OpenAIChatCompletion): ), event_normalizer=self._make_event_normalizer(), usage_meter=self._make_usage_meter(), - translation_session=realtime_mode == "translation", ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 8f020901714..1cc1c6dccab 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -35,11 +35,15 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): try: sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: - stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout) + stream_response: Final = await openai_aclient.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options + **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, + ) return {}, stream_response - raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( - **sdk_data, timeout=timeout - ) # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options + **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, + ) headers: Final = dict(raw_response.headers) parsed_response: Final = raw_response.parse() @@ -56,10 +60,14 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): try: sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: - stream_response: Final = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) + stream_response: Final = openai_client.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options + **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, + ) return None, stream_response - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( - **sdk_data, timeout=timeout + raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options + **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, ) headers: Final = dict(raw_response.headers) response: Final = raw_response.parse() diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 29fc1ceef68..ff6cb99dabc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,26 +1,4 @@ { - "azure/gpt-live-transcribe": { - "input_cost_per_second": 0.0002833333333333333, - "litellm_provider": "azure", - "mode": "audio_transcription", - "provider_specific_entry": { - "realtime_ga_only": 1 - }, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", - "supported_endpoints": [ - "/v1/realtime", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "audio", - "text" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_native_streaming": true - }, "azure/gpt-realtime-2.1-2026-07-07": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, @@ -101,33 +79,6 @@ "supports_prompt_caching": true, "supports_reasoning": true }, - "azure/gpt-realtime-translate": { - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "provider_specific_entry": { - "realtime_ga_only": 1 - }, - "output_cost_per_second": 0.0005666666666666667, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", - "supported_endpoints": [ - "/v1/realtime/translations", - "/v1/realtime/translations/client_secrets", - "/v1/realtime/translations/calls" - ], - "supported_modalities": [ - "audio" - ], - "supported_output_modalities": [ - "audio", - "text" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_native_streaming": true - }, "azure/gpt-realtime-translate-2026-05-06": { "litellm_provider": "azure", "max_input_tokens": 32000, @@ -228,31 +179,6 @@ "max_output_tokens": 4096, "max_tokens": 4096 }, - "azure/gpt-transcribe": { - "input_cost_per_second": 7.5e-05, - "litellm_provider": "azure", - "mode": "audio_transcription", - "provider_specific_entry": { - "realtime_ga_only": 1, - "transcription_deployment_api": 1, - "transcription_json_only": 1 - }, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", - "supported_endpoints": [ - "/v1/audio/transcriptions", - "/v1/realtime", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "audio", - "text" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_native_streaming": true - }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -71174,6 +71100,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/realtime" @@ -71232,7 +71161,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-transcribe": { "deprecation_date": "2028-02-01", @@ -71242,7 +71175,8 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" + "/v1/realtime/transcription_sessions", + "/v1/realtime" ], "supported_modalities": [ "text", @@ -71251,17 +71185,22 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-realtime-translate": { "deprecation_date": "2027-11-09", - "input_cost_per_second": 0.000566666666667, "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_modalities": [ "audio" ], @@ -71270,7 +71209,17 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.000566666666667, + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supports_native_streaming": true }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b3d9c04f4da..4f9ba0e6ed8 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -33147,6 +33147,7 @@ }, "/openai/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post", "responses": { "200": { @@ -33166,6 +33167,7 @@ }, "/openai/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post", "responses": { "200": { @@ -33219,15 +33221,31 @@ }, "/openai/v1/realtime/translations/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { "schema": {} + }, + "application/sdp": { + "schema": { + "type": "string" + } } }, - "description": "Successful Response" + "description": "WebRTC SDP answer" } }, "summary": "Proxy Realtime Calls", @@ -33238,7 +33256,441 @@ }, "/openai/v1/realtime/translations/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "enum": [ + "minimal", + "low", + "medium", + "high", + "xhigh" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "enum": [ + "realtime", + "transcription", + "translation" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -53335,6 +53787,7 @@ "paths": { "/openai/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post_2", "responses": { "200": { @@ -53354,6 +53807,7 @@ }, "/openai/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post_2", "responses": { "200": { @@ -53407,15 +53861,31 @@ }, "/openai/v1/realtime/translations/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post_2", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { "schema": {} + }, + "application/sdp": { + "schema": { + "type": "string" + } } }, - "description": "Successful Response" + "description": "WebRTC SDP answer" } }, "summary": "Proxy Realtime Calls", @@ -53426,7 +53896,441 @@ }, "/openai/v1/realtime/translations/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post_2", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "enum": [ + "minimal", + "low", + "medium", + "high", + "xhigh" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "enum": [ + "realtime", + "transcription", + "translation" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -53452,6 +54356,7 @@ }, "/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_realtime_calls_post", "responses": { "200": { @@ -53471,6 +54376,7 @@ }, "/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_realtime_client_secrets_post", "responses": { "200": { @@ -53524,15 +54430,31 @@ }, "/realtime/translations/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_realtime_translations_calls_post", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { "schema": {} + }, + "application/sdp": { + "schema": { + "type": "string" + } } }, - "description": "Successful Response" + "description": "WebRTC SDP answer" } }, "summary": "Proxy Realtime Calls", @@ -53543,7 +54465,441 @@ }, "/realtime/translations/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_realtime_translations_client_secrets_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "enum": [ + "minimal", + "low", + "medium", + "high", + "xhigh" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "enum": [ + "realtime", + "transcription", + "translation" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -53569,6 +54925,7 @@ }, "/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_v1_realtime_calls_post", "responses": { "200": { @@ -53588,6 +54945,7 @@ }, "/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_v1_realtime_client_secrets_post", "responses": { "200": { @@ -53641,15 +54999,31 @@ }, "/v1/realtime/translations/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_v1_realtime_translations_calls_post", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { "schema": {} + }, + "application/sdp": { + "schema": { + "type": "string" + } } }, - "description": "Successful Response" + "description": "WebRTC SDP answer" } }, "summary": "Proxy Realtime Calls", @@ -53660,7 +55034,441 @@ }, "/v1/realtime/translations/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_v1_realtime_translations_client_secrets_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "enum": [ + "minimal", + "low", + "medium", + "high", + "xhigh" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "enum": [ + "realtime", + "transcription", + "translation" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 997a73be66a..c054ef65daf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13720,6 +13720,7 @@ async def realtime_websocket_endpoint( model=model, intent=intent, guardrails=guardrails, + is_translation=is_translation, ) @@ -13733,6 +13734,7 @@ async def _route_realtime_websocket_session( guardrails: str | None, request_path: str | None = None, live_session_start: Mapping[str, object] | None = None, + is_translation: bool = False, ) -> None: # Only use explicit parameters, not all query params query_model: Final = _resolve_realtime_upstream_query_model( diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index c28439a6395..44afaab7706 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -13,6 +13,7 @@ from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import can_key_call_resolved_model from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -49,6 +50,20 @@ _NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE: Final = ( ) +_TRANSLATION_SECRET_BODY: Final = { + "requestBody": inline_request_body( + RealtimeClientSecretRequest, + {"model": "gpt-realtime-translate", "session": {"type": "translation", "model": "gpt-realtime-translate"}}, + ) +} +_TRANSLATION_SDP_BODY: Final = { + "requestBody": {"required": True, "content": {"application/sdp": {"schema": {"type": "string"}}}} +} +_TRANSLATION_SDP_RESPONSES: Final = { + 200: {"description": "WebRTC SDP answer", "content": {"application/sdp": {"schema": {"type": "string"}}}} +} + + def _enforce_non_billable_realtime_protocol_gate(general_settings: Mapping[str, object]) -> None: if general_settings.get(_NON_BILLABLE_REALTIME_PROTOCOL_SETTING) is True: return @@ -289,16 +304,19 @@ def _decode_realtime_token_payload( ) @router.post( "/v1/realtime/translations/client_secrets", + openapi_extra=_TRANSLATION_SECRET_BODY, dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) @router.post( "/realtime/translations/client_secrets", + openapi_extra=_TRANSLATION_SECRET_BODY, dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) @router.post( "/openai/v1/realtime/translations/client_secrets", + openapi_extra=_TRANSLATION_SECRET_BODY, dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) @@ -307,6 +325,7 @@ async def create_realtime_client_secret( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ) -> RealtimeClientSecretResponse: + """Create a short-lived client token for a Realtime conversation, transcription, or translation session""" from litellm.proxy.proxy_server import ( add_litellm_data_to_request, general_settings, @@ -453,20 +472,27 @@ async def create_realtime_client_secret( ) @router.post( "/v1/realtime/translations/calls", + openapi_extra=_TRANSLATION_SDP_BODY, + responses=_TRANSLATION_SDP_RESPONSES, tags=["realtime"], ) @router.post( "/realtime/translations/calls", + openapi_extra=_TRANSLATION_SDP_BODY, + responses=_TRANSLATION_SDP_RESPONSES, tags=["realtime"], ) @router.post( "/openai/v1/realtime/translations/calls", + openapi_extra=_TRANSLATION_SDP_BODY, + responses=_TRANSLATION_SDP_RESPONSES, tags=["realtime"], ) async def proxy_realtime_calls( request: Request, fastapi_response: Response, ) -> Response: + """Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets""" from litellm.proxy.proxy_server import ( add_litellm_data_to_request, general_settings, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 2c946a3dd91..bb967aa1dba 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -7,10 +7,9 @@ from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast -from typing_extensions import ReadOnly, TypedDict, assert_never - import httpx from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict, assert_never import litellm from litellm.constants import ( @@ -19,6 +18,7 @@ from litellm.constants import ( REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig @@ -82,12 +82,9 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: if session.get("type") == "transcription": - audio = session.get("audio") - audio = audio if isinstance(audio, dict) else {} - audio_input = audio.get("input") - audio_input = audio_input if isinstance(audio_input, dict) else {} - transcription = audio_input.get("transcription") - transcription = transcription if isinstance(transcription, dict) else {} + audio: Final = as_str_mapping(session.get("audio")) or {} + audio_input: Final = as_str_mapping(audio.get("input")) or {} + transcription: Final = as_str_mapping(audio_input.get("transcription")) or {} return { **session, "audio": { @@ -663,7 +660,6 @@ async def _arealtime( timeout: float | None = None, query_params: RealtimeQueryParams | None = None, live_session_start: Mapping[str, object] | None = None, - realtime_mode: str = "realtime", **kwargs, ) -> None: diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index fafa93152e6..37e1b57192c 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,7 +1,7 @@ from collections.abc import Mapping, Sequence -from typing import Any, Final, Literal +from typing import Any, Literal, TypeAlias -from pydantic import BaseModel, ConfigDict +from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -13,7 +13,7 @@ from .llms.openai import ( ) ALL_DELTA_TYPES = Literal["text", "audio"] -RealtimeSessionType: Final = Literal["realtime", "transcription", "translation"] +RealtimeSessionType: TypeAlias = Literal["realtime", "transcription", "translation"] class RealtimeResponseTransformInput(TypedDict): @@ -68,8 +68,8 @@ class RealtimeExpiresAfter(LiteLLMBaseModel): seconds: int | None = None -class RealtimeAudioTranscriptionConfig(BaseModel): - model_config = ConfigDict(extra="allow") +class RealtimeAudioTranscriptionConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) model: str | None = None delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None @@ -79,8 +79,8 @@ class RealtimeAudioTranscriptionConfig(BaseModel): prompt: str | None = None -class RealtimeAudioInputConfig(BaseModel): - model_config = ConfigDict(extra="allow") +class RealtimeAudioInputConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) format: str | Mapping[str, object] | None = None noise_reduction: Mapping[str, object] | None = None @@ -88,22 +88,22 @@ class RealtimeAudioInputConfig(BaseModel): turn_detection: Mapping[str, object] | None = None -class RealtimeAudioOutputConfig(BaseModel): - model_config = ConfigDict(extra="allow") +class RealtimeAudioOutputConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) format: str | Mapping[str, object] | None = None language: str | None = None voice: str | Mapping[str, object] | None = None -class RealtimeSessionAudioConfig(BaseModel): - model_config = ConfigDict(extra="allow") +class RealtimeSessionAudioConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) input: RealtimeAudioInputConfig | None = None output: RealtimeAudioOutputConfig | None = None -class RealtimeSessionConfig(BaseModel): +class RealtimeSessionConfig(LiteLLMBaseModel): """ Session configuration nested inside the client_secrets request body. @@ -112,7 +112,7 @@ class RealtimeSessionConfig(BaseModel): Extra/unknown fields are passed through unchanged. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) type: RealtimeSessionType | None = None model: str | None = None @@ -166,7 +166,7 @@ class RealtimeTranscriptionSessionRequest(LiteLLMBaseModel): unchanged to the provider. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) # LiteLLM-only routing hint — stripped before forwarding upstream. model: str | None = None @@ -191,7 +191,7 @@ class RealtimeTranscriptionSessionResponse(LiteLLMBaseModel): ephemeral key. Unknown fields pass through unchanged. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) client_secret: dict[str, object] | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a9ef1f04a8e..5a40e2eba8d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2880,7 +2880,9 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None -class TranscriptionDetectedLanguage(BaseModel): +class TranscriptionDetectedLanguage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + code: str @@ -2900,7 +2902,13 @@ class TranscriptionResponse(OpenAIObject): def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params - def __init__(self, text=None, usage=None, languages=None, **kwargs) -> None: # noqa: ANN003 # OpenAI-compatible response accepts provider extension fields + def __init__( + self, + text: str | None = None, + usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None, + languages: Sequence[TranscriptionDetectedLanguage] | None = None, + **kwargs: object, + ) -> None: super().__init__(text=text, usage=usage, languages=languages, **kwargs) def set_audio_transcription_duration(self, duration: float) -> None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 29fc1ceef68..ff6cb99dabc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,26 +1,4 @@ { - "azure/gpt-live-transcribe": { - "input_cost_per_second": 0.0002833333333333333, - "litellm_provider": "azure", - "mode": "audio_transcription", - "provider_specific_entry": { - "realtime_ga_only": 1 - }, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", - "supported_endpoints": [ - "/v1/realtime", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "audio", - "text" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_native_streaming": true - }, "azure/gpt-realtime-2.1-2026-07-07": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, @@ -101,33 +79,6 @@ "supports_prompt_caching": true, "supports_reasoning": true }, - "azure/gpt-realtime-translate": { - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "provider_specific_entry": { - "realtime_ga_only": 1 - }, - "output_cost_per_second": 0.0005666666666666667, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", - "supported_endpoints": [ - "/v1/realtime/translations", - "/v1/realtime/translations/client_secrets", - "/v1/realtime/translations/calls" - ], - "supported_modalities": [ - "audio" - ], - "supported_output_modalities": [ - "audio", - "text" - ], - "supports_audio_input": true, - "supports_audio_output": true, - "supports_native_streaming": true - }, "azure/gpt-realtime-translate-2026-05-06": { "litellm_provider": "azure", "max_input_tokens": 32000, @@ -228,31 +179,6 @@ "max_output_tokens": 4096, "max_tokens": 4096 }, - "azure/gpt-transcribe": { - "input_cost_per_second": 7.5e-05, - "litellm_provider": "azure", - "mode": "audio_transcription", - "provider_specific_entry": { - "realtime_ga_only": 1, - "transcription_deployment_api": 1, - "transcription_json_only": 1 - }, - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", - "supported_endpoints": [ - "/v1/audio/transcriptions", - "/v1/realtime", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "audio", - "text" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true, - "supports_native_streaming": true - }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -71174,6 +71100,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/realtime" @@ -71232,7 +71161,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-transcribe": { "deprecation_date": "2028-02-01", @@ -71242,7 +71175,8 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" + "/v1/realtime/transcription_sessions", + "/v1/realtime" ], "supported_modalities": [ "text", @@ -71251,17 +71185,22 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-realtime-translate": { "deprecation_date": "2027-11-09", - "input_cost_per_second": 0.000566666666667, "litellm_provider": "azure", "max_input_tokens": 32000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_modalities": [ "audio" ], @@ -71270,7 +71209,17 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.000566666666667, + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supports_native_streaming": true }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, diff --git a/scripts/check_openapi_docs.py b/scripts/check_openapi_docs.py index 26bb034bfa6..8dd83d12912 100755 --- a/scripts/check_openapi_docs.py +++ b/scripts/check_openapi_docs.py @@ -30,18 +30,18 @@ def _as_mapping(node: object) -> Mapping[str, object]: return node if isinstance(node, Mapping) else {} -def _json_schema(container: object) -> object: +def _content_schema(container: object) -> object: content: Final = _as_mapping(_as_mapping(container).get("content")) - return _as_mapping(content.get("application/json")).get("schema") + return next((schema for media in content.values() if (schema := _as_mapping(media).get("schema"))), None) def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]: - if not _json_schema(operation.get("requestBody")): + if not _content_schema(operation.get("requestBody")): yield "OA001" description: Final = operation.get("description") if not isinstance(description, str) or not description.strip(): yield "OA002" - if not _json_schema(_as_mapping(operation.get("responses")).get("200")): + if not _content_schema(_as_mapping(operation.get("responses")).get("200")): yield "OA003" diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index d9fab04b7ae..5e8d19fd8a2 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3066,7 +3066,7 @@ def test_translation_audio_duration_is_finalized_once(event_type: str): streaming = RealTimeStreaming( websocket=MagicMock(), backend_ws=MagicMock(), - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) @@ -3106,7 +3106,7 @@ async def test_translation_disconnect_bills_sent_input_audio( streaming: Final = RealTimeStreaming( websocket=_ga_client_ws(), backend_ws=backend, - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) @@ -3143,7 +3143,7 @@ async def test_translation_client_format_updates_do_not_change_billed_duration(r backend: Final = MagicMock() backend.send = AsyncMock() streaming: Final = RealTimeStreaming( - websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True ) streaming._capture_translation_output_audio( { @@ -3184,7 +3184,7 @@ async def test_translation_confirmed_format_changes_preserve_previous_audio_dura backend: Final = MagicMock() backend.send = AsyncMock() streaming: Final = RealTimeStreaming( - websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True ) await streaming._send_to_backend( json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(48000)).decode()}) @@ -3214,7 +3214,7 @@ async def test_translation_confirmed_format_changes_preserve_previous_audio_dura @pytest.mark.parametrize("rate", (1, 0, -1, float("nan"), float("inf"), 10**400)) def test_translation_invalid_provider_format_preserves_audio_duration(rate: int | float) -> None: streaming: Final = RealTimeStreaming( - websocket=_ga_client_ws(), backend_ws=MagicMock(), logging_obj=MagicMock(), translation_session=True + websocket=_ga_client_ws(), backend_ws=MagicMock(), logging_obj=_mock_logging(), translation_session=True ) output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} streaming._capture_translation_output_audio(output_event) @@ -3232,7 +3232,7 @@ async def test_translation_failed_audio_send_is_not_billed() -> None: backend: Final = MagicMock() backend.send = AsyncMock(side_effect=RuntimeError("send failed")) streaming: Final = RealTimeStreaming( - websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True ) with pytest.raises(RuntimeError, match="send failed"): @@ -3267,7 +3267,7 @@ async def test_translation_rejects_applicable_guardrails_before_forwarding_audio streaming: Final = RealTimeStreaming( websocket=client, backend_ws=backend, - logging_obj=MagicMock(), + logging_obj=_mock_logging(), translation_session=True, request_data={"litellm_metadata": {"guardrails": ["speech-policy"] if requested else []}}, ) @@ -3371,7 +3371,7 @@ async def test_translation_terminal_usage_fills_only_missing_input_duration( backend: Final = MagicMock() backend.send = AsyncMock() streaming: Final = RealTimeStreaming( - websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True ) await streaming._send_to_backend( json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) @@ -3399,7 +3399,7 @@ def test_translation_audio_duration_uses_session_output_format(): streaming = RealTimeStreaming( websocket=MagicMock(), backend_ws=MagicMock(), - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) @@ -3424,7 +3424,7 @@ def test_translation_does_not_duplicate_provider_duration_usage(): streaming = RealTimeStreaming( websocket=MagicMock(), backend_ws=MagicMock(), - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) @@ -3444,7 +3444,7 @@ def test_translation_prefers_provider_duration_over_audio_byte_estimate(): streaming = RealTimeStreaming( websocket=MagicMock(), backend_ws=MagicMock(), - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) @@ -3474,7 +3474,7 @@ def test_translation_preserves_input_only_provider_usage( streaming = RealTimeStreaming( websocket=MagicMock(), backend_ws=MagicMock(), - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) @@ -3496,7 +3496,7 @@ def test_translation_retained_input_only_close_event_bills_captured_output(monke streaming = RealTimeStreaming( websocket=MagicMock(), backend_ws=MagicMock(), - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) @@ -3520,7 +3520,7 @@ def test_translation_malformed_provider_duration_uses_captured_audio(malformed_o streaming = RealTimeStreaming( websocket=MagicMock(), backend_ws=MagicMock(), - logging_obj=MagicMock(), + logging_obj=_mock_logging(), model="gpt-realtime-translate", translation_session=True, ) diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index 14e8bb34801..4e456e364d2 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -6,6 +6,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES + from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from litellm.types.realtime import RealtimeQueryParams @@ -517,7 +519,7 @@ async def test_translation_websocket_uses_direct_transport(beta_client: bool) -> ) ) websocket.close = AsyncMock() - logging_obj: Final = MagicMock() + logging_obj: Final = MagicMock(dispatch_success_handlers=AsyncMock(), dispatch_failure_handlers=AsyncMock()) handler: Final = OpenAIRealtime() expected_url: Final = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" assert ( diff --git a/tests/unit/proxy/common_utils/test_realtime_cache.py b/tests/unit/proxy/common_utils/test_realtime_cache.py index 9750a17ae14..10dc436f274 100644 --- a/tests/unit/proxy/common_utils/test_realtime_cache.py +++ b/tests/unit/proxy/common_utils/test_realtime_cache.py @@ -47,7 +47,7 @@ def test_realtime_request_body_preserves_default_model_authorization(is_translat from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model - body: Final = TypeAdapter(Mapping[str, object]).validate_json(_realtime_request_body(None)) + body: Final = TypeAdapter(Mapping[str, object]).validate_json(realtime_request_body(None)) route: Final = "/v1/realtime/translations" if is_translation else "/v1/realtime" intent: Final = None if is_translation else "transcription" admitted: Final = managed_inference_request(route, body, {}, None, intent=intent) @@ -58,7 +58,7 @@ def test_realtime_request_body_preserves_default_model_authorization(is_translat @pytest.mark.parametrize("model", (None, "requested", 'model"with\\quotes', "model\nwith-newline")) def test_realtime_request_body_preserves_model_names(model: str | None) -> None: - body: Final = TypeAdapter(Mapping[str, object]).validate_json(_realtime_request_body(model)) + body: Final = TypeAdapter(Mapping[str, object]).validate_json(realtime_request_body(model)) assert body["model"] == model diff --git a/tests/unit/test_check_openapi_docs.py b/tests/unit/test_check_openapi_docs.py index d7122541416..ef8cf0e9dbd 100644 --- a/tests/unit/test_check_openapi_docs.py +++ b/tests/unit/test_check_openapi_docs.py @@ -2,6 +2,8 @@ from typing import Final +import pytest + import check_openapi_docs as checker _BODY: Final = {"content": {"application/json": {"schema": {"type": "object"}}}} @@ -9,8 +11,17 @@ _RESPONSES: Final = {"200": {"content": {"application/json": {"schema": {"$ref": _DOCUMENTED: Final = {"description": "Does a thing", "requestBody": _BODY, "responses": _RESPONSES} -def test_a_fully_documented_post_has_no_findings() -> None: - assert checker.violations({"paths": {"/v1/thing": {"post": _DOCUMENTED}}}) == () +@pytest.mark.parametrize( + ("request_media_type", "response_media_type"), + (("application/json", "application/json"), ("application/sdp", "application/sdp"), ("multipart/form-data", "audio/wav")), +) +def test_a_fully_documented_post_has_no_findings(request_media_type: str, response_media_type: str) -> None: + operation: Final = { + **_DOCUMENTED, + "requestBody": {"content": {request_media_type: {"schema": {"type": "string"}}}}, + "responses": {"200": {"content": {response_media_type: {"schema": {"type": "string"}}}}}, + } + assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == () def test_each_missing_piece_is_its_own_rule() -> None: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8d918949310..80368fe1e73 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11009,7 +11009,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_openai_v1_realtime_calls_post"]; delete?: never; options?: never; @@ -11026,7 +11029,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_openai_v1_realtime_client_secrets_post"]; delete?: never; options?: never; @@ -11087,7 +11093,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_openai_v1_realtime_translations_calls_post"]; delete?: never; options?: never; @@ -11104,7 +11113,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post"]; delete?: never; options?: never; @@ -14037,7 +14049,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_realtime_calls_post"]; delete?: never; options?: never; @@ -14054,7 +14069,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_realtime_client_secrets_post"]; delete?: never; options?: never; @@ -14115,7 +14133,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_realtime_translations_calls_post"]; delete?: never; options?: never; @@ -14132,7 +14153,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_realtime_translations_client_secrets_post"]; delete?: never; options?: never; @@ -21793,7 +21817,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_v1_realtime_calls_post"]; delete?: never; options?: never; @@ -21810,7 +21837,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_v1_realtime_client_secrets_post"]; delete?: never; options?: never; @@ -21871,7 +21901,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_v1_realtime_translations_calls_post"]; delete?: never; options?: never; @@ -21888,7 +21921,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_v1_realtime_translations_client_secrets_post"]; delete?: never; options?: never; @@ -66788,7 +66824,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_3: { + websocket_realtime_websocket_endpoint_get_6: { parameters: { query?: never; header?: never; @@ -66891,15 +66927,20 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + "application/sdp": string; + }; + }; responses: { - /** @description Successful Response */ + /** @description WebRTC SDP answer */ 200: { headers: { [name: string]: unknown; }; content: { "application/json": unknown; + "application/sdp": string; }; }; }; @@ -66911,7 +66952,112 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "gpt-realtime-translate", + * "session": { + * "model": "gpt-realtime-translate", + * "type": "translation" + * } + * } + */ + "application/json": { + expires_after?: { + /** + * Anchor + * @default created_at + */ + anchor?: string | null; + /** Seconds */ + seconds?: number | null; + } | null; + /** Model */ + model?: string | null; + session?: ({ + audio?: ({ + input?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Noise Reduction */ + noise_reduction?: { + [key: string]: unknown; + } | null; + transcription?: ({ + /** Delay */ + delay?: ("minimal" | "low" | "medium" | "high" | "xhigh") | null; + /** Keywords */ + keywords?: string[] | null; + /** Language */ + language?: string | null; + /** Languages */ + languages?: string[] | null; + /** Model */ + model?: string | null; + /** Prompt */ + prompt?: string | null; + } & { + [key: string]: unknown; + }) | null; + /** Turn Detection */ + turn_detection?: { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + output?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Language */ + language?: string | null; + /** Voice */ + voice?: string | { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + } & { + [key: string]: unknown; + }) | null; + /** Include */ + include?: string[] | null; + /** Instructions */ + instructions?: string | null; + /** Max Output Tokens */ + max_output_tokens?: number | string | null; + /** Model */ + model?: string | null; + /** Output Modalities */ + output_modalities?: string[] | null; + /** Prompt */ + prompt?: { + [key: string]: unknown; + } | null; + /** Tool Choice */ + tool_choice?: unknown | null; + /** Tools */ + tools?: { + [key: string]: unknown; + }[] | null; + /** Tracing */ + tracing?: unknown | null; + /** Truncation */ + truncation?: unknown | null; + /** Type */ + type?: ("realtime" | "transcription" | "translation") | null; + } & { + [key: string]: unknown; + }) | null; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -70304,15 +70450,20 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + "application/sdp": string; + }; + }; responses: { - /** @description Successful Response */ + /** @description WebRTC SDP answer */ 200: { headers: { [name: string]: unknown; }; content: { "application/json": unknown; + "application/sdp": string; }; }; }; @@ -70324,7 +70475,112 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "gpt-realtime-translate", + * "session": { + * "model": "gpt-realtime-translate", + * "type": "translation" + * } + * } + */ + "application/json": { + expires_after?: { + /** + * Anchor + * @default created_at + */ + anchor?: string | null; + /** Seconds */ + seconds?: number | null; + } | null; + /** Model */ + model?: string | null; + session?: ({ + audio?: ({ + input?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Noise Reduction */ + noise_reduction?: { + [key: string]: unknown; + } | null; + transcription?: ({ + /** Delay */ + delay?: ("minimal" | "low" | "medium" | "high" | "xhigh") | null; + /** Keywords */ + keywords?: string[] | null; + /** Language */ + language?: string | null; + /** Languages */ + languages?: string[] | null; + /** Model */ + model?: string | null; + /** Prompt */ + prompt?: string | null; + } & { + [key: string]: unknown; + }) | null; + /** Turn Detection */ + turn_detection?: { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + output?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Language */ + language?: string | null; + /** Voice */ + voice?: string | { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + } & { + [key: string]: unknown; + }) | null; + /** Include */ + include?: string[] | null; + /** Instructions */ + instructions?: string | null; + /** Max Output Tokens */ + max_output_tokens?: number | string | null; + /** Model */ + model?: string | null; + /** Output Modalities */ + output_modalities?: string[] | null; + /** Prompt */ + prompt?: { + [key: string]: unknown; + } | null; + /** Tool Choice */ + tool_choice?: unknown | null; + /** Tools */ + tools?: { + [key: string]: unknown; + }[] | null; + /** Tracing */ + tracing?: unknown | null; + /** Truncation */ + truncation?: unknown | null; + /** Type */ + type?: ("realtime" | "transcription" | "translation") | null; + } & { + [key: string]: unknown; + }) | null; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -81264,15 +81520,20 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + "application/sdp": string; + }; + }; responses: { - /** @description Successful Response */ + /** @description WebRTC SDP answer */ 200: { headers: { [name: string]: unknown; }; content: { "application/json": unknown; + "application/sdp": string; }; }; }; @@ -81284,7 +81545,112 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "gpt-realtime-translate", + * "session": { + * "model": "gpt-realtime-translate", + * "type": "translation" + * } + * } + */ + "application/json": { + expires_after?: { + /** + * Anchor + * @default created_at + */ + anchor?: string | null; + /** Seconds */ + seconds?: number | null; + } | null; + /** Model */ + model?: string | null; + session?: ({ + audio?: ({ + input?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Noise Reduction */ + noise_reduction?: { + [key: string]: unknown; + } | null; + transcription?: ({ + /** Delay */ + delay?: ("minimal" | "low" | "medium" | "high" | "xhigh") | null; + /** Keywords */ + keywords?: string[] | null; + /** Language */ + language?: string | null; + /** Languages */ + languages?: string[] | null; + /** Model */ + model?: string | null; + /** Prompt */ + prompt?: string | null; + } & { + [key: string]: unknown; + }) | null; + /** Turn Detection */ + turn_detection?: { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + output?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Language */ + language?: string | null; + /** Voice */ + voice?: string | { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + } & { + [key: string]: unknown; + }) | null; + /** Include */ + include?: string[] | null; + /** Instructions */ + instructions?: string | null; + /** Max Output Tokens */ + max_output_tokens?: number | string | null; + /** Model */ + model?: string | null; + /** Output Modalities */ + output_modalities?: string[] | null; + /** Prompt */ + prompt?: { + [key: string]: unknown; + } | null; + /** Tool Choice */ + tool_choice?: unknown | null; + /** Tools */ + tools?: { + [key: string]: unknown; + }[] | null; + /** Tracing */ + tracing?: unknown | null; + /** Truncation */ + truncation?: unknown | null; + /** Type */ + type?: ("realtime" | "transcription" | "translation") | null; + } & { + [key: string]: unknown; + }) | null; + }; + }; + }; responses: { /** @description Successful Response */ 200: { From d8262a441b544613a3e4bc7247866d18f1430a0a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Fri, 16 Oct 2026 11:13:03 -0500 Subject: [PATCH 27/45] fix(audio): validate provider responses at SDK boundaries --- litellm/cost_calculator.py | 12 ++-- litellm/litellm_core_utils/litellm_logging.py | 7 ++- .../convert_dict_to_response.py | 6 +- .../litellm_core_utils/realtime_streaming.py | 7 +-- litellm/llms/azure/audio_transcriptions.py | 59 ++++++++++++++----- .../audio_transcription/transformation.py | 9 ++- litellm/llms/openai/realtime/handler.py | 30 ++++++---- .../transcriptions/gpt_transformation.py | 8 +-- litellm/llms/openai/transcriptions/handler.py | 58 ++++++++++++------ litellm/main.py | 2 +- litellm/proxy/realtime_endpoints/endpoints.py | 2 +- litellm/realtime_api/main.py | 23 +++++--- litellm/types/utils.py | 6 +- litellm/utils.py | 3 - 14 files changed, 148 insertions(+), 84 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 296b8ad2ae6..8f2b1749be6 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -3781,7 +3781,7 @@ def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: def _transcription_usage_cost( - usage: dict, + usage: Mapping[str, object], model_info: ModelInfo | None, override_info: ModelInfo | None = None, ) -> float: @@ -3790,13 +3790,13 @@ def _transcription_usage_cost( usage_type: Final = usage.get("type") if usage_type == "duration": - seconds: Final = usage.get("seconds") or 0.0 + seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0 return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info) if usage_type == "tokens": - input_token_details: Final = usage.get("input_token_details") or {} - audio_tokens: Final = input_token_details.get("audio_tokens") or 0 - text_tokens: Final = input_token_details.get("text_tokens") or 0 - output_tokens: Final = usage.get("output_tokens") or 0 + input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {} + audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0 + text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0 + output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0 audio_cost: Final = float(audio_tokens) * _transcription_rate( ("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f5d8ebe53e4..17634f93d98 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3243,7 +3243,12 @@ class Logging(LiteLLMLoggingBaseClass): try: ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: ( - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None ) = None if "complete_streaming_response" in self.model_call_details and not for_async_request: return # break out of this. diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 8de9c9810fd..e49526ad505 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -949,9 +949,9 @@ def convert_to_model_response_object( if "languages" in response_object and response_object["languages"] is not None: transcription_response: Final = model_response_object - transcription_response.languages = tuple( - TranscriptionDetectedLanguage.model_validate(language) for language in response_object["languages"] - ) + transcription_response.languages = TypeAdapter( + tuple[TranscriptionDetectedLanguage, ...] + ).validate_python(response_object["languages"]) if "usage" in response_object and response_object["usage"] is not None: tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index e5b30c2a0e4..ab8e19eb4ca 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -694,10 +694,9 @@ class RealTimeStreaming: for event in self.messages: if event.get("type") != "session.closed": continue - if ( - isinstance(event_usage := event.get("usage"), dict) - and normalized_audio_duration_seconds(event_usage.get("output_seconds")) is not None - ): + if (event_usage := as_str_mapping(event.get("usage"))) is not None and normalized_audio_duration_seconds( + event_usage.get("output_seconds") + ) is not None: self._translation_usage_finalized = True return if self._translation_output_seconds == 0 and self._translation_input_seconds == 0: diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 3600ac9e5e1..c47e6832a4e 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,14 +1,22 @@ from collections.abc import Coroutine, Mapping -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final, cast -from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI, Stream +from openai.types.audio import TranscriptionStreamEvent +from openai.types.audio.transcription_create_params import ( + TranscriptionCreateParamsBase, + TranscriptionCreateParamsStreaming, +) from pydantic import BaseModel import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data +from litellm.llms.base_llm.audio_transcription.transformation import ( + sdk_compatible_transcription_request_data, + sdk_compatible_transcription_response, +) from litellm.types.utils import FileTypes from litellm.utils import ( TranscriptionResponse, @@ -41,7 +49,11 @@ class AzureAudioTranscription(AzureChatCompletion): atranscription: bool = False, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: + ) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ): data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) model_info: Final = as_str_mapping(litellm.model_cost.get(f"azure/{model}")) @@ -101,13 +113,23 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - response: Final = azure_client.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options - **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options - timeout=timeout, - ) - if data.get("stream") is True: - return response + return azure_client.audio.transcriptions.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsStreaming, sdk_data + ), + timeout=timeout, + ) + + response: Final = sdk_compatible_transcription_response( + azure_client.audio.transcriptions.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsBase, {key: value for key, value in sdk_data.items() if key != "stream"} + ), + stream=False, + timeout=timeout, + ) + ) if isinstance(response, BaseModel): stringified_response = response.model_dump() @@ -145,7 +167,7 @@ class AzureAudioTranscription(AzureChatCompletion): max_retries=None, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse: + ) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: response = None try: async_azure_client: Final = self.get_azure_openai_client( @@ -176,18 +198,23 @@ class AzureAudioTranscription(AzureChatCompletion): ) if data.get("stream") is True: - return await async_azure_client.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options - **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + return await async_azure_client.audio.transcriptions.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsStreaming, data + ), timeout=timeout, ) - raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options - **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"} + ), + stream=False, timeout=timeout, ) headers: Final = dict(raw_response.headers) - response = raw_response.parse() + response = sdk_compatible_transcription_response(raw_response.parse()) if isinstance(response, BaseModel): stringified_response = response.model_dump() diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 5d4efd32491..b2b99d92f3d 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final import httpx -from pydantic import TypeAdapter +from pydantic import BaseModel, TypeAdapter from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.chat.transformation import BaseConfig @@ -15,6 +15,7 @@ from litellm.types.llms.openai import ( from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse _TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object]) +_TRANSCRIPTION_TEXT: Final = TypeAdapter(str) def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]: @@ -43,6 +44,12 @@ else: LiteLLMLoggingObj = Any +def sdk_compatible_transcription_response(response: object) -> BaseModel | str: + if isinstance(response, BaseModel): + return response + return _TRANSCRIPTION_TEXT.validate_python(response, strict=True) + + @dataclass class AudioTranscriptionRequestData: """ diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index c39cb74f224..b41336f70a3 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -15,6 +15,8 @@ from openai.resources.realtime.realtime import ( AsyncRealtimeConnection, AsyncRealtimeConnectionManager, ) +from openai.types.websocket_connection_options import WebSocketConnectionOptions +from typing_extensions import NotRequired, ReadOnly from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES @@ -33,6 +35,11 @@ from ..common_utils import is_openai_backed_api_base from ..openai import OpenAIChatCompletion +class _RealtimeConnectionOptions(WebSocketConnectionOptions): + open_timeout: NotRequired[ReadOnly[float]] + ssl: NotRequired[ReadOnly[bool | str | ssl.SSLContext | None]] + + class OpenAIRealtimeConnectionAdapter: def __init__(self, connection: AsyncRealtimeConnection) -> None: self._connection = connection @@ -190,20 +197,21 @@ class OpenAIRealtime(OpenAIChatCompletion): headers: Mapping[str, str], timeout: float | None, realtime_mode: str, - ssl_config: object, + ssl_config: bool | str | ssl.SSLContext | None, client: object | None, url: str, ) -> AbstractAsyncContextManager[object]: import websockets + timeout_options: Final[_RealtimeConnectionOptions] = {"open_timeout": timeout} if timeout is not None else {} + ssl_options: Final[_RealtimeConnectionOptions] = {"ssl": ssl_config} if url.startswith("wss://") else {} + connection_options: Final[_RealtimeConnectionOptions] = { + "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + **ssl_options, + **timeout_options, + } if realtime_mode == "translation" or not isinstance(client, AsyncOpenAI): - return websockets.connect( - url, - additional_headers=headers, - max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - ssl=ssl_config, - **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), - ) + return websockets.connect(url, additional_headers=headers, **connection_options) openai_client: Final = client model_query: Final = query_params.get("model") extra_query: Final = MappingProxyType({key: value for key, value in query_params.items() if key != "model"}) @@ -212,11 +220,7 @@ class OpenAIRealtime(OpenAIChatCompletion): model=sdk_model, extra_query=extra_query, extra_headers=headers, - websocket_connection_options={ - "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - **(MappingProxyType({"ssl": ssl_config}) if url.startswith("wss://") else MappingProxyType({})), - **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), - }, + websocket_connection_options=connection_options, max_retries=0, ) return OpenAIRealtimeSDKConnectionManager(sdk_connection_manager) diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index cbeb7a6d811..99fbb6b96c8 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -26,8 +26,8 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): self, model: str, audio_file: FileTypes, - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request @@ -53,8 +53,8 @@ class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionCon self, model: str, audio_file: FileTypes, - optional_params: dict, # mutable-ok: base transformation interface supplies a mutable request payload - litellm_params: dict, # mutable-ok: base transformation interface supplies mutable provider parameters + optional_params: dict[str, object], + litellm_params: dict[str, object], ) -> AudioTranscriptionRequestData: data: Final = { "model": model, diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 1cc1c6dccab..3a55ab01404 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,8 +1,13 @@ -from collections.abc import Mapping +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final, Optional, cast import httpx -from openai import AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream +from openai.types.audio import TranscriptionStreamEvent +from openai.types.audio.transcription_create_params import ( + TranscriptionCreateParamsBase, + TranscriptionCreateParamsStreaming, +) from pydantic import BaseModel if TYPE_CHECKING: @@ -13,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, sdk_compatible_transcription_request_data, + sdk_compatible_transcription_response, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -29,23 +35,28 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): async def make_openai_audio_transcriptions_request( self, openai_aclient: AsyncOpenAI, - data: dict, + data: Mapping[str, object], timeout: float | httpx.Timeout, ): try: sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: - stream_response: Final = await openai_aclient.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options - **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + stream_response: Final = await openai_aclient.audio.transcriptions.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsStreaming, sdk_data + ), timeout=timeout, ) return {}, stream_response - raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options - **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsBase, {key: value for key, value in sdk_data.items() if key != "stream"} + ), + stream=False, timeout=timeout, ) headers: Final = dict(raw_response.headers) - parsed_response: Final = raw_response.parse() + parsed_response: Final = sdk_compatible_transcription_response(raw_response.parse()) return headers, parsed_response except Exception as e: @@ -54,23 +65,28 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): def make_sync_openai_audio_transcriptions_request( self, openai_client: OpenAI, - data: dict, + data: Mapping[str, object], timeout: float | httpx.Timeout, ): try: sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: - stream_response: Final = openai_client.audio.transcriptions.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options - **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + stream_response: Final = openai_client.audio.transcriptions.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsStreaming, sdk_data + ), timeout=timeout, ) - return None, stream_response - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( # pyright: ignore[reportCallIssue] # SDK overloads cannot model dynamic transcription options - **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + return {}, stream_response + raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( + **cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsBase, {key: value for key, value in sdk_data.items() if key != "stream"} + ), + stream=False, timeout=timeout, ) headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() + response: Final = sdk_compatible_transcription_response(raw_response.parse()) return headers, response except Exception as e: raise e @@ -91,7 +107,11 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): atranscription: bool = False, provider_config: BaseAudioTranscriptionConfig | None = None, shared_session: Optional["ClientSession"] = None, - ) -> TranscriptionResponse: + ) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ): """ Handle audio transcription request """ @@ -149,7 +169,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): ) logging_obj.model_call_details["response_headers"] = headers - if data.get("stream") is True: + if isinstance(response, Stream): return response if isinstance(response, BaseModel): @@ -177,7 +197,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): async def async_audio_transcriptions( self, audio_file: FileTypes, - data: dict, + data: Mapping[str, object], model_response: TranscriptionResponse, timeout: float, logging_obj: LiteLLMLoggingObj, @@ -216,7 +236,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, ) logging_obj.model_call_details["response_headers"] = headers - if data.get("stream") is True: + if isinstance(response, AsyncStream): return response if isinstance(response, BaseModel): stringified_response = response.model_dump() diff --git a/litellm/main.py b/litellm/main.py index 09880ec5768..6953e1d8cbb 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -799,7 +799,7 @@ def _model_response_from_cached_dict(cached_response_dict: Mapping[str, object]) def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TranscriptionResponse: - return TranscriptionResponse(**cached_response_dict) + return TranscriptionResponse.model_validate(cached_response_dict) async def _async_streaming(response, model, custom_llm_provider, args): diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 44afaab7706..f86f99c8ff3 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -59,7 +59,7 @@ _TRANSLATION_SECRET_BODY: Final = { _TRANSLATION_SDP_BODY: Final = { "requestBody": {"required": True, "content": {"application/sdp": {"schema": {"type": "string"}}}} } -_TRANSLATION_SDP_RESPONSES: Final = { +_TRANSLATION_SDP_RESPONSES: Final[dict[int | str, dict[str, object]]] = { 200: {"description": "WebRTC SDP answer", "content": {"application/sdp": {"schema": {"type": "string"}}}} } diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index bb967aa1dba..9821f46045f 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast import httpx +from openai import AsyncOpenAI from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never @@ -45,7 +46,7 @@ from ..litellm_core_utils.realtime_streaming import client_websocket_scope from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed -from ..llms.custom_httpx.http_handler import realtime_ssl_for_url +from ..llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, realtime_ssl_for_url from ..llms.openai.live.handler import OpenAILiveSessions from ..llms.openai.realtime.handler import OpenAIRealtime from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async @@ -266,7 +267,7 @@ async def acreate_realtime_translation_client_secret( session: Mapping[str, Any] | None = None, expires_after: Mapping[str, Any] | None = None, timeout: float | None = None, - **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options + **kwargs: object, # kwargs-ok: public API preserves provider-specific request options ) -> httpx.Response: requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate" session_config: Final = RealtimeSessionConfig.model_validate( @@ -284,7 +285,7 @@ async def acreate_realtime_translation_client_secret( litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") if not isinstance(litellm_logging_obj, LiteLLMLogging): raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") - litellm_params: Final = GenericLiteLLMParams(**kwargs) + litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs) model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( model=requested_model_name, @@ -301,7 +302,7 @@ async def acreate_realtime_translation_client_secret( provider_config=provider_config, litellm_params=litellm_params, resolved_api_key=resolved_api_key, - extra_headers=kwargs.get("extra_headers"), + extra_headers=as_str_mapping(kwargs.get("extra_headers")), ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, @@ -328,7 +329,9 @@ async def acreate_realtime_translation_client_secret( provider_config=provider_config, model=model_name, extra_headers=resolved_extra_headers, - client=kwargs.get("client"), + client=client + if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler, AsyncOpenAI)) + else None, api_version=litellm_params.api_version, use_openai_sdk=custom_llm_provider == "openai", ) @@ -477,13 +480,13 @@ async def arealtime_translation_calls( model: str | None = None, session: Mapping[str, Any] | None = None, timeout: float | None = None, - **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options + **kwargs: object, # kwargs-ok: public API preserves provider-specific request options ) -> httpx.Response: requested_model_name: Final = model or "gpt-realtime-translate" litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") if not isinstance(litellm_logging_obj, LiteLLMLogging): raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") - litellm_params: Final = GenericLiteLLMParams(**kwargs) + litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs) model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( model=requested_model_name, @@ -523,8 +526,10 @@ async def arealtime_translation_calls( provider_config=provider_config, model=model_name, session_config=session_config, - extra_headers=kwargs.get("extra_headers"), - client=kwargs.get("client"), + extra_headers=as_str_mapping(kwargs.get("extra_headers")), + client=client + if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler, AsyncOpenAI)) + else None, api_version=litellm_params.api_version, translation=True, use_openai_sdk=custom_llm_provider == "openai", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5a40e2eba8d..80f31f382b7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2905,9 +2905,9 @@ class TranscriptionResponse(OpenAIObject): def __init__( self, text: str | None = None, - usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None, - languages: Sequence[TranscriptionDetectedLanguage] | None = None, - **kwargs: object, + usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | Mapping[str, object] | None = None, + languages: Sequence[TranscriptionDetectedLanguage | Mapping[str, object]] | None = None, + **kwargs: object, # kwargs-ok: OpenAI-compatible responses preserve provider extension fields ) -> None: super().__init__(text=text, usage=usage, languages=languages, **kwargs) diff --git a/litellm/utils.py b/litellm/utils.py index 4dfd59540de..211f58f3104 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1431,9 +1431,6 @@ def function_setup( applied_guardrails=applied_guardrails, supports_correlation_logging=is_async_call, ) - if logging_obj is None: - raise RuntimeError("LiteLLM logging initialization returned no logger") - ## check if metadata is passed in litellm_params: Final[dict[str, object]] = {"api_base": ""} if "metadata" in kwargs: From 98009dcce0bbd1c3ee4aca88ba4868c0acd6cf90 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Fri, 16 Oct 2026 11:23:01 -0500 Subject: [PATCH 28/45] fix(realtime): preserve minimum OpenAI SDK compatibility --- litellm/llms/openai/realtime/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index b41336f70a3..b640d4d9c14 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -15,7 +15,7 @@ from openai.resources.realtime.realtime import ( AsyncRealtimeConnection, AsyncRealtimeConnectionManager, ) -from openai.types.websocket_connection_options import WebSocketConnectionOptions +from openai.types.websocket_connection_options import WebsocketConnectionOptions as WebSocketConnectionOptions from typing_extensions import NotRequired, ReadOnly from litellm._logging import redact_string, verbose_logger From cb710d6b4e88986f59b34808f2b3db41d82e7811 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Fri, 16 Oct 2026 11:36:44 -0500 Subject: [PATCH 29/45] fix(realtime): preserve deferred Live session authorization --- litellm/proxy/auth/user_api_key_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index e22d8058c0b..d81083c5d8b 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1184,7 +1184,7 @@ async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth: async def user_api_key_auth_live_websocket(websocket: WebSocket) -> UserAPIKeyAuth: - return await user_api_key_auth_websocket_for_model(websocket, model=None) + return await user_api_key_auth_websocket_for_model(websocket, model="") def _has_required_baggage_tag_mapping() -> bool: From 4ffff0a57895a78fb559df3a3a2c059b62229cfa Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Mon, 19 Oct 2026 23:24:56 -0500 Subject: [PATCH 30/45] fix(realtime): preserve upstream authentication and transcription behavior --- litellm/cost_calculator.py | 15 +++++------ .../transcriptions/gpt_transformation.py | 1 - litellm/realtime_api/main.py | 4 ++- .../llms/openai/realtime/test_translation.py | 11 +++++--- .../transcriptions/test_gpt_transcribe.py | 2 +- tests/unit/realtime_api/test_main.py | 26 ++++++++++++++----- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8f2b1749be6..6ee11918760 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1888,14 +1888,13 @@ def completion_cost( # the response attribute (for verbose_json responses that # naturally include duration from the provider). _hidden = getattr(completion_response, "_hidden_params", {}) or {} - provider_duration = _get_transcription_usage_duration(completion_response) - audio_transcription_file_duration = ( - provider_duration - if provider_duration is not None - else _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), - ) + audio_transcription_file_duration = _hidden.get( + "audio_transcription_duration", + getattr(completion_response, "duration", 0.0), + ) or ( + (_get_transcription_usage_duration(completion_response) or 0.0) + if _bills_provider_transcription_seconds(requested_model, custom_llm_provider) + else 0.0 ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index 99fbb6b96c8..dc072f72525 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -59,7 +59,6 @@ class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionCon data: Final = { "model": model, "file": audio_file, - "response_format": "json", **optional_params, } return AudioTranscriptionRequestData(data=data) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 9821f46045f..f6e63e515cb 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -322,7 +322,9 @@ async def acreate_realtime_translation_client_secret( request_data["session"].pop("type", None) return await base_llm_http_handler.async_realtime_translation_client_secret_handler( api_base=resolved_api_base, - api_key=resolved_api_key, + api_key=await _realtime_http_bearer_token( + custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params + ), request_data=request_data, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py index 06af49f26ca..97e9f822253 100644 --- a/tests/unit/llms/openai/realtime/test_translation.py +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -1,5 +1,6 @@ import gzip import json +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx @@ -78,14 +79,16 @@ def test_openai_translation_calls_url(): ) -def test_azure_translation_urls_use_ga_paths(): - config = AzureRealtimeHTTPConfig() +@pytest.mark.parametrize("suffix", ("", "/", "/openai", "/openai/v1/")) +def test_azure_translation_urls_use_ga_paths(suffix: str): + config: Final = AzureRealtimeHTTPConfig() + api_base: Final = f"https://example.openai.azure.com{suffix}" assert ( - config.get_translation_client_secret_url("https://example.openai.azure.com", "translate-deployment") + config.get_translation_client_secret_url(api_base, "translate-deployment") == "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets" ) assert ( - config.get_translation_calls_url("https://example.openai.azure.com", "translate-deployment") + config.get_translation_calls_url(api_base, "translate-deployment") == "https://example.openai.azure.com/openai/v1/realtime/translations/calls" ) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 535b52a17ee..ccc176e6451 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -38,7 +38,7 @@ def test_gpt_transcribe_config_uses_native_parameters_and_json(): request = config.transform_audio_transcription_request( model="gpt-transcribe", audio_file=audio_file, - optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True}, + optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True, "response_format": "json"}, litellm_params={}, ) assert request.data["response_format"] == "json" diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index b6aca93cd67..dac63bf1d8e 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -2,6 +2,7 @@ import asyncio import json import time from collections.abc import Awaitable, Callable, Mapping +from datetime import datetime, timezone from pathlib import Path from types import TracebackType from typing import Final @@ -13,6 +14,7 @@ import respx import litellm from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai.workload_identity import _workload_identity_auth @@ -1188,20 +1190,32 @@ class _GatewayThatRecordsTheAuthorization: @pytest.mark.asyncio @pytest.mark.parametrize( - "route", - [realtime_main.acreate_realtime_client_secret, realtime_main.acreate_realtime_transcription_session], - ids=["client_secrets", "transcription_sessions"], + ("route", "model"), + [ + (realtime_main.acreate_realtime_client_secret, "gpt-realtime"), + (realtime_main.acreate_realtime_transcription_session, "gpt-realtime"), + (realtime_main.acreate_realtime_translation_client_secret, "gpt-realtime-translate"), + ], + ids=["client_secrets", "transcription_sessions", "translation_client_secrets"], ) @respx.mock async def test_openai_realtime_http_routes_send_the_federated_bearer_for_a_keyless_deployment( - federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]] + federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]], model: str ) -> None: _mock_token_exchange() gateway: Final = _GatewayThatRecordsTheAuthorization() await route.__wrapped__( - model="gpt-realtime", + model=model, custom_llm_provider="openai", - litellm_logging_obj=FakeLogging(), + litellm_logging_obj=Logging( + model=model, + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-federated-realtime", + function_id="test-federated-realtime", + ), client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), **federated_deployment, ) From 8212c9498b5fb8b22e3074c5f80d062b3b2ab125 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Mon, 19 Oct 2026 23:28:50 -0500 Subject: [PATCH 31/45] fix(audio): scope provider duration precedence to configured models --- litellm/cost_calculator.py | 29 ++++++++++++++----- ...odel_prices_and_context_window_backup.json | 6 ++-- model_prices_and_context_window.json | 6 ++-- .../test_transcription_duration_hidden.py | 3 +- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6ee11918760..d2540115475 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1888,13 +1888,28 @@ def completion_cost( # the response attribute (for verbose_json responses that # naturally include duration from the provider). _hidden = getattr(completion_response, "_hidden_params", {}) or {} - audio_transcription_file_duration = _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), - ) or ( - (_get_transcription_usage_duration(completion_response) or 0.0) - if _bills_provider_transcription_seconds(requested_model, custom_llm_provider) - else 0.0 + provider_duration: Final = _get_transcription_usage_duration(completion_response) + transcription_info: Final = ( + _get_model_info_or_none(requested_model, custom_llm_provider) + if requested_model is not None and custom_llm_provider is not None + else None + ) + transcription_metadata: Final = ( + as_str_mapping(transcription_info.get("provider_specific_entry")) + if transcription_info is not None + else None + ) + audio_transcription_file_duration = ( + provider_duration + if provider_duration is not None + and transcription_metadata is not None + and transcription_metadata.get("transcription_usage_duration") == 1 + else _hidden.get("audio_transcription_duration", getattr(completion_response, "duration", 0.0)) + or ( + (provider_duration or 0.0) + if _bills_provider_transcription_seconds(requested_model, custom_llm_provider) + else 0.0 + ) ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ff6cb99dabc..163d501ec01 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63357,7 +63357,8 @@ "litellm_provider": "openai", "mode": "audio_transcription", "provider_specific_entry": { - "transcription_json_only": 1 + "transcription_json_only": 1, + "transcription_usage_duration": 1 }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -71189,7 +71190,8 @@ "provider_specific_entry": { "realtime_ga_only": 1, "transcription_deployment_api": 1, - "transcription_json_only": 1 + "transcription_json_only": 1, + "transcription_usage_duration": 1 }, "supports_native_streaming": true }, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ff6cb99dabc..163d501ec01 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63357,7 +63357,8 @@ "litellm_provider": "openai", "mode": "audio_transcription", "provider_specific_entry": { - "transcription_json_only": 1 + "transcription_json_only": 1, + "transcription_usage_duration": 1 }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ @@ -71189,7 +71190,8 @@ "provider_specific_entry": { "realtime_ga_only": 1, "transcription_deployment_api": 1, - "transcription_json_only": 1 + "transcription_json_only": 1, + "transcription_usage_duration": 1 }, "supports_native_streaming": true }, diff --git a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py index a15ba6d7543..b55ceab633c 100644 --- a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py +++ b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py @@ -165,6 +165,7 @@ class TestCostCalculatorReadsDurationFromHiddenParams: ): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setitem(litellm.model_cost, model, {**litellm.model_cost[model], "input_cost_per_second": 0.0125}) response = TranscriptionResponse( text="test", usage={"type": "duration", "seconds": 60.0}, @@ -178,7 +179,7 @@ class TestCostCalculatorReadsDurationFromHiddenParams: custom_llm_provider=provider, ) - assert actual == pytest.approx(0.0045) + assert actual == pytest.approx(60.0 * 0.0125) @patch("litellm.cost_calculator.openai_cost_per_second") def test_completion_cost_uses_hidden_params_duration(self, mock_cost_fn): From 04b704192c4d6f830020e534aeb27b386650835c Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Mon, 19 Oct 2026 23:32:18 -0500 Subject: [PATCH 32/45] fix(typing): avoid Final declarations inside transcription cost loop --- litellm/cost_calculator.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index d2540115475..67ccc83aa27 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1888,21 +1888,20 @@ def completion_cost( # the response attribute (for verbose_json responses that # naturally include duration from the provider). _hidden = getattr(completion_response, "_hidden_params", {}) or {} - provider_duration: Final = _get_transcription_usage_duration(completion_response) - transcription_info: Final = ( + provider_duration, transcription_info = ( + _get_transcription_usage_duration(completion_response), _get_model_info_or_none(requested_model, custom_llm_provider) if requested_model is not None and custom_llm_provider is not None - else None - ) - transcription_metadata: Final = ( - as_str_mapping(transcription_info.get("provider_specific_entry")) - if transcription_info is not None - else None + else None, ) audio_transcription_file_duration = ( provider_duration if provider_duration is not None - and transcription_metadata is not None + and transcription_info is not None + and ( + transcription_metadata := as_str_mapping(transcription_info.get("provider_specific_entry")) + ) + is not None and transcription_metadata.get("transcription_usage_duration") == 1 else _hidden.get("audio_transcription_duration", getattr(completion_response, "duration", 0.0)) or ( From 19bcd5c58483c9d28288d9c2e5a9ce9f05672509 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Mon, 19 Oct 2026 23:39:03 -0500 Subject: [PATCH 33/45] refactor(audio): reuse transcription mapping and duration resolution --- litellm/cost_calculator.py | 57 +++++++++---------- .../transcriptions/gpt_transformation.py | 14 ----- 2 files changed, 28 insertions(+), 43 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 67ccc83aa27..09a7c2a61e4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -352,13 +352,36 @@ def _transcription_uses_token_pricing( return (prompt_tokens_val > 0) or (completion_tokens_val > 0) -def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool: +def _transcription_duration_for_cost( + completion_response: object, model: str | None, custom_llm_provider: str | None +) -> float: + provider_duration: Final = _get_transcription_usage_duration(completion_response) + transcription_info: Final = ( + _get_model_info_or_none(model, custom_llm_provider) + if model is not None and custom_llm_provider is not None + else None + ) + transcription_metadata: Final = ( + as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None + ) + if ( + provider_duration is not None + and transcription_metadata is not None + and transcription_metadata.get("transcription_usage_duration") == 1 + ): + return provider_duration + hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {} + measured_duration: Final = normalized_audio_duration_seconds( + hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration")) + ) + if measured_duration is not None and measured_duration > 0: + return measured_duration if model is None or custom_llm_provider != LlmProviders.OPENAI.value: - return False + return 0.0 transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI ) - return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) + return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"] @@ -1883,32 +1906,8 @@ def completion_cost( elif call_type in _SPEECH_CALL_TYPES: prompt_characters = litellm.utils.count_characters(text=prompt) elif call_type in _TRANSCRIPTION_CALL_TYPES: - # Check _hidden_params first (duration stored there to - # avoid polluting the response body), then fall back to - # the response attribute (for verbose_json responses that - # naturally include duration from the provider). - _hidden = getattr(completion_response, "_hidden_params", {}) or {} - provider_duration, transcription_info = ( - _get_transcription_usage_duration(completion_response), - _get_model_info_or_none(requested_model, custom_llm_provider) - if requested_model is not None and custom_llm_provider is not None - else None, - ) - audio_transcription_file_duration = ( - provider_duration - if provider_duration is not None - and transcription_info is not None - and ( - transcription_metadata := as_str_mapping(transcription_info.get("provider_specific_entry")) - ) - is not None - and transcription_metadata.get("transcription_usage_duration") == 1 - else _hidden.get("audio_transcription_duration", getattr(completion_response, "duration", 0.0)) - or ( - (provider_duration or 0.0) - if _bills_provider_transcription_seconds(requested_model, custom_llm_provider) - else 0.0 - ) + audio_transcription_file_duration = _transcription_duration_for_cost( + completion_response, requested_model, custom_llm_provider ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index dc072f72525..b9eaa204567 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -48,17 +48,3 @@ class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionCon "languages", "stream", ] - - def transform_audio_transcription_request( - self, - model: str, - audio_file: FileTypes, - optional_params: dict[str, object], - litellm_params: dict[str, object], - ) -> AudioTranscriptionRequestData: - data: Final = { - "model": model, - "file": audio_file, - **optional_params, - } - return AudioTranscriptionRequestData(data=data) From 8855c1385eb40e1a7fa766132b5d09bbd06cb4c3 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Mon, 19 Oct 2026 23:59:53 -0500 Subject: [PATCH 34/45] fix(realtime): enforce model budgets and preserve translation routing --- .../litellm_core_utils/realtime_streaming.py | 27 +++++++--- litellm/llms/azure/audio_transcriptions.py | 2 +- litellm/proxy/auth/user_api_key_auth.py | 9 +++- litellm/realtime_api/main.py | 2 + .../test_realtime_streaming.py | 44 ++++++++------- .../llms/azure/test_audio_transcriptions.py | 2 +- .../test_user_api_key_auth_request_flow.py | 53 +++++++++++++++++++ tests/unit/realtime_api/test_main.py | 41 ++++++++++++++ 8 files changed, 151 insertions(+), 29 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index ab8e19eb4ca..8b866666c6a 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -727,10 +727,25 @@ class RealTimeStreaming: backend, False if the provider transformation produced no output and the message was effectively dropped. """ - message = await self._apply_nested_transcription_model_policy(message) + from litellm.proxy._types import ProxyException + + try: + authorized_message: Final = await self._apply_nested_transcription_model_policy(message) + except ProxyException as denied: + denial_message: Final = redact_internal_details_from_client_message(denied.message) + try: + await self.websocket.send_text(realtime_error_event(denial_message, error_type="permission_error")) + finally: + await self.websocket.close( + code=1008, + reason=websocket_close_reason( + denial_message, fallback="Nested transcription model is not authorized" + ), + ) + return False if self.provider_config: transformed: Final = self.provider_config.transform_realtime_request( - message, self.model, self.session_configuration_request + authorized_message, self.model, self.session_configuration_request ) sent = False for msg in transformed: @@ -766,14 +781,14 @@ class RealTimeStreaming: self._content_sent_after_setup = True sent = True if sent: - self._capture_translation_input_audio(message) + self._capture_translation_input_audio(authorized_message) return sent - await self.backend_ws.send(message) + await self.backend_ws.send(authorized_message) if self._usage_meter is not None: - sent_event: Final = self._parse_backend_event(message) + sent_event: Final = self._parse_backend_event(authorized_message) if sent_event is not None: self._usage_meter.observe_client_event(sent_event) - self._capture_translation_input_audio(message) + self._capture_translation_input_audio(authorized_message) return True async def _apply_nested_transcription_model_policy(self, message: str) -> str: diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index c47e6832a4e..90c240a611c 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -60,7 +60,7 @@ class AzureAudioTranscription(AzureChatCompletion): provider_specific_entry: Final = ( as_str_mapping(model_info.get("provider_specific_entry")) if model_info is not None else None ) - requires_deployment_api: Final = model_info is None or ( + requires_deployment_api: Final = ( provider_specific_entry is not None and provider_specific_entry.get("transcription_deployment_api") == 1 ) resolved_api_version: Final = ( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index d81083c5d8b..b51da241318 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -144,6 +144,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg populate_request_with_path_params, read_raw_json_body, read_request_body, + resolve_realtime_route_model, rewrite_request_model, safe_get_request_headers, safe_get_request_query_params, @@ -1180,7 +1181,13 @@ def _apply_budget_limits_to_end_user_params( async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth: - return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model")) + route_path: Final[object] = websocket.scope.get("path") + model: Final = resolve_realtime_route_model( + model=websocket.query_params.get("model"), + intent=websocket.query_params.get("intent"), + is_translation=isinstance(route_path, str) and route_path.endswith("/realtime/translations"), + ) + return await user_api_key_auth_websocket_for_model(websocket, model=model) async def user_api_key_auth_live_websocket(websocket: WebSocket) -> UserAPIKeyAuth: diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index f6e63e515cb..c5add19291d 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -289,6 +289,7 @@ async def acreate_realtime_translation_client_secret( model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( model=requested_model_name, + custom_llm_provider=litellm_params.custom_llm_provider, api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) @@ -492,6 +493,7 @@ async def arealtime_translation_calls( model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( model=requested_model_name, + custom_llm_provider=litellm_params.custom_llm_provider, api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 5e8d19fd8a2..4c91b0b1707 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -879,9 +879,7 @@ async def test_usage_meter_snapshot_stored_in_model_call_details(): backend_ws.send = AsyncMock() backend_ws.recv = AsyncMock( side_effect=[ - json.dumps( - {"type": "session.created", "session": {"turn_detection": {"type": None}}} - ).encode(), + json.dumps({"type": "session.created", "session": {"turn_detection": {"type": None}}}).encode(), json.dumps( {"type": "response.output_audio.delta", "delta": base64.b64encode(b"x" * 24000).decode()} ).encode(), @@ -1211,10 +1209,11 @@ async def test_transcription_session_update_enforces_authorized_nested_model(): @pytest.mark.asyncio async def test_translation_session_update_rejects_disallowed_nested_transcription_model() -> None: - backend_ws = MagicMock() + client_ws: Final = MagicMock(send_text=AsyncMock(), close=AsyncMock()) + backend_ws: Final = MagicMock() backend_ws.send = AsyncMock() - streaming = RealTimeStreaming( - MagicMock(), + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, MagicMock(), model="gpt-realtime-translate", @@ -1222,24 +1221,29 @@ async def test_translation_session_update_rejects_disallowed_nested_transcriptio translation_session=True, ) - with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available"): - await streaming._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": { - "type": "translation", - "audio": { - "input": { - "transcription": {"model": "gpt-live-transcribe"}, - } - }, + sent: Final = await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe"}, + } }, - } - ) + }, + } ) + ) + assert sent is False backend_ws.send.assert_not_awaited() assert streaming._is_transcription_session is False + error: Final = json.loads(client_ws.send_text.await_args.args[0]) + assert error["type"] == "error" + assert error["error"]["type"] == "permission_error" + assert "gpt-live-transcribe" in error["error"]["message"] + assert client_ws.close.await_args.kwargs["code"] == 1008 @pytest.mark.asyncio diff --git a/tests/unit/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py index d8f1a290d5e..f8a294c7953 100644 --- a/tests/unit/llms/azure/test_audio_transcriptions.py +++ b/tests/unit/llms/azure/test_audio_transcriptions.py @@ -172,7 +172,7 @@ async def test_azure_transcription_parses_response_formats( [ ("whisper-1", "/openai/v1/audio/transcriptions"), ("gpt-transcribe", "/openai/deployments/gpt-transcribe/audio/transcriptions"), - ("custom-transcribe-deployment", "/openai/deployments/custom-transcribe-deployment/audio/transcriptions"), + ("custom-transcribe-deployment", "/openai/v1/audio/transcriptions"), ], ) def test_azure_transcription_alias_uses_model_route( diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 4cdd7fefd28..6ea5693bbe6 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -11936,6 +11936,59 @@ async def test_websocket_auth_hands_the_reservation_to_the_socket_state(): assert websocket.scope["state"]["budget_reservation"] is reservation +@pytest.mark.asyncio +@pytest.mark.parametrize( + "path", ("/realtime/translations", "/v1/realtime/translations", "/openai/v1/realtime/translations") +) +@pytest.mark.parametrize("query", (b"", b"model=gpt-realtime-translate"), ids=("default_model", "explicit_model")) +async def test_translation_websocket_auth_enforces_the_default_model_budget( + monkeypatch: pytest.MonkeyPatch, path: str, query: bytes +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + + model: Final = "gpt-realtime-translate" + token: Final = UserAPIKeyAuth( + token="test-key", + models=[model], + model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0}) + with pytest.raises(litellm.BudgetExceededError): + await limiter.is_key_within_model_budget(user_api_key_dict=token, model=model) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": query, + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + with pytest.raises(HTTPException) as denied: + await user_api_key_auth_websocket(websocket) + + assert denied.value.status_code == 403 + assert model in str(denied.value.detail) + assert "budget" in str(denied.value.detail).lower() + + @pytest.mark.asyncio async def test_admission_and_budget_reservation_read_the_key_spend_counter_with_one_redis_mget(): from fastapi import Request diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index dac63bf1d8e..26280ad4523 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1222,6 +1222,47 @@ async def test_openai_realtime_http_routes_send_the_federated_bearer_for_a_keyle assert gateway.authorization == "Bearer federated-bearer" +@pytest.mark.asyncio +@pytest.mark.parametrize("flow", ("client_secret", "calls")) +async def test_translation_http_routes_preserve_explicit_azure_provider(flow: str) -> None: + expected_path: Final = "/openai/v1/realtime/translations/" + { + "client_secret": "client_secrets", + "calls": "calls", + }[flow] + + def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.host == "azure-deployment.example" + assert request.url.path == expected_path + return httpx.Response(200, json={"value": "ek-test", "routed_path": request.url.path}) + + params: Final[Mapping[str, object]] = { + "model": "translation-deployment", + "custom_llm_provider": "azure", + "api_base": "https://azure-deployment.example/openai/v1/", + "api_key": "test-key", + "client": AsyncHTTPHandler(transport=httpx.MockTransport(send_response)), + "litellm_logging_obj": Logging( + model="translation-deployment", + messages=[], + stream=False, + call_type="acreate_realtime_translation_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-translation-routing", + function_id="test-translation-routing", + ), + } + response: Final = await ( + realtime_main.acreate_realtime_translation_client_secret.__wrapped__(**params) + if flow == "client_secret" + else realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek-test", sdp_body=b"v=0", **params + ) + ) + + assert response.status_code == 200 + assert response.json() == {"value": "ek-test", "routed_path": expected_path} + + @pytest.mark.asyncio @respx.mock async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_keyless_deployment( From 2501523ecd9d31c1766e2d7312993ac9b6b005ae Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:16:00 -0700 Subject: [PATCH 35/45] fix(audio): log streamed transcripts whose done event reports detected languages --- .../audio_utils/transcription_streaming.py | 11 +++++++++-- .../llms/openai/transcriptions/test_gpt_transcribe.py | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index cf2681f5d36..719909d66ea 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -11,8 +11,11 @@ from openai.types.audio import ( TranscriptionTextDeltaEvent, TranscriptionTextDoneEvent, ) +from pydantic import TypeAdapter -from litellm.types.utils import TranscriptionResponse +from litellm.types.utils import TranscriptionDetectedLanguage, TranscriptionResponse + +_DETECTED_LANGUAGES: Final = TypeAdapter(list[TranscriptionDetectedLanguage]) class TranscriptionStreamLogging(Protocol): @@ -72,7 +75,11 @@ class _TranscriptionEventCollector: response: Final = TranscriptionResponse( text=done_event.text if done_event is not None else "".join(self.text_deltas), usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, - languages=([language.model_dump() for language in done_languages] if done_languages is not None else None), + languages=( + _DETECTED_LANGUAGES.validate_python(done_languages, from_attributes=True) + if done_languages is not None + else None + ), ) if self.duration is not None: response.set_audio_transcription_duration(self.duration) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index ccc176e6451..dc8feac29e9 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -75,7 +75,7 @@ def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: content=( 'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' 'data: {"type":"transcript.text.done","text":"hello world",' - '"usage":{"type":"duration","seconds":2.5}}\n\n' + '"usage":{"type":"duration","seconds":2.5},"languages":[{"code":"en"}]}\n\n' ), ) @@ -97,6 +97,7 @@ def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: logged_response: Final = logging_obj.success_handler.call_args.args[0] assert logged_response.text == "hello world" assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5} + assert logged_response.model_dump()["languages"] == [{"code": "en"}] logging_obj.failure_handler.assert_not_called() From 572bf65013bb96316cd952c8b3bff9d5bb824037 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:32:55 -0700 Subject: [PATCH 36/45] fix(audio): pass stream, languages and keywords through unchecked for non-OpenAI transcription providers Providers without a streaming transcription handler got UnsupportedParamsError for stream=True, languages and keywords, where the merge base forwarded them as provider-specific params. Only openai and azure validate them now. --- litellm/utils.py | 22 +++++++++++----------- tests/unit/test_utils.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 211f58f3104..d9aeaa53a6b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4033,6 +4033,9 @@ def _get_non_default_params(passed_params: dict, default_params: dict, additiona return non_default_params +_OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS: Final = frozenset(("languages", "keywords", "stream")) + + def get_optional_params_transcription( model: str, custom_llm_provider: str, @@ -4060,16 +4063,13 @@ def get_optional_params_transcription( for k, v in special_params.items(): passed_params[k] = v - default_params: Final = { - "language": None, - "languages": None, - "keywords": None, - "prompt": None, - "response_format": None, - "temperature": None, # openai defaults this to 0 - "timestamp_granularities": None, - "stream": None, - } + sends_through_openai_sdk: Final = custom_llm_provider in ("openai", "azure") + transcription_params: Final = ( + OPENAI_TRANSCRIPTION_PARAMS + if sends_through_openai_sdk + else [param for param in OPENAI_TRANSCRIPTION_PARAMS if param not in _OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS] + ) + default_params: Final = {param: None for param in (*transcription_params, "prompt", "temperature")} non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} optional_params = {} @@ -4122,7 +4122,7 @@ def get_optional_params_transcription( optional_params=optional_params, passed_params=passed_params, custom_llm_provider=custom_llm_provider, - openai_params=OPENAI_TRANSCRIPTION_PARAMS, + openai_params=transcription_params, additional_drop_params=kwargs.get("additional_drop_params", None), ) extra_body: Final = optional_params.get("extra_body") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b0d2546774e..8a34e00966e 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -103,6 +103,7 @@ from litellm.utils import ( client, get_non_default_completion_params, get_optional_params_image_gen, + get_optional_params_transcription, get_prompt_cache_min_tokens, get_supported_openai_params, get_token_count, @@ -10498,6 +10499,25 @@ def test_get_provider_audio_transcription_config_resolves_for_every_provider() - assert isinstance(configs[LlmProviders.OPENAI], litellm.OpenAIWhisperAudioTranscriptionConfig) + +@pytest.mark.parametrize( + ("custom_llm_provider", "model"), + [ + ("groq", "whisper-large-v3"), + ("deepgram", "nova-3"), + ("mistral", "voxtral-mini-latest"), + ("elevenlabs", "scribe_v1"), + ("fireworks_ai", "whisper-v3"), + ], +) +def test_openai_sdk_transcription_params_pass_through_to_other_providers(custom_llm_provider: str, model: str) -> None: + hints: Final = {"stream": True, "languages": ["en"], "keywords": ["LiteLLM"]} + + params: Final = get_optional_params_transcription(model=model, custom_llm_provider=custom_llm_provider, **hints) + + forwarded: Final = {**params.get("extra_body", {}), **params} + assert {key: forwarded.get(key) for key in hints} == hints + @pytest.mark.parametrize( ("model", "optional_params", "expected_response_format"), [ From b321d67030d91dc2afe33ba9391fa668b2d375ce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:35:04 -0700 Subject: [PATCH 37/45] fix(audio): bill streamed gpt-transcribe from the provider's reported duration The stream path computed cost after logging had replaced the transcription's duration usage, so it fell back to the measured file length while the unstreamed call billed the provider's seconds. --- litellm/litellm_core_utils/litellm_logging.py | 10 +- .../transcriptions/test_gpt_transcribe.py | 161 ++++++++++++++++++ 2 files changed, 169 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 17634f93d98..bfebc9b7667 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2857,6 +2857,10 @@ class Logging(LiteLLMLoggingBaseClass): setattr(result, "usage", transformed_usage) return result + @staticmethod + def _stream_cost_source(received_result: object, assembled_response: object) -> object: + return received_result if isinstance(received_result, TranscriptionResponse) else assembled_response + def _with_client_facing_stream_model( self, response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, @@ -3229,6 +3233,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit) if not self.should_run_logging(event_type="sync_success"): # prevent double logging return + received_result: Final = result start_time, end_time, result, request_logging_payload = self._prepare_success_logging( start_time=start_time, end_time=end_time, @@ -3264,7 +3269,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["complete_streaming_response"] = complete_streaming_response self._surface_response_headers_from_result(complete_streaming_response) self.model_call_details["response_cost"] = self.response_cost_calculator( - result=complete_streaming_response + result=self._stream_cost_source(received_result, complete_streaming_response) ) self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) @@ -3802,6 +3807,7 @@ class Logging(LiteLLMLoggingBaseClass): kwargs=self.model_call_details, messages=self.model_call_details.get("messages") ) ) + received_result: Final = result start_time, end_time, result = self._success_handler_helper_fn( start_time=start_time, end_time=end_time, @@ -3843,7 +3849,7 @@ class Logging(LiteLLMLoggingBaseClass): get_base_model_from_metadata(model_call_details=self.model_call_details) # base_model defaults to None if not set on model_info self.model_call_details["response_cost"] = self.response_cost_calculator( - result=complete_streaming_response + result=self._stream_cost_source(received_result, complete_streaming_response) ) verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"]) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index dc8feac29e9..10c145ba6df 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -362,3 +362,164 @@ async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): assert response.text == "hello" await client.close() + + +def _one_second_wav() -> io.BytesIO: + audio_file: Final = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + audio_file.seek(0) + return audio_file + + +class _SuccessCostRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.async_costs: Final[asyncio.Queue[float | None]] = asyncio.Queue() + self.sync_costs: Final[queue.Queue[float | None]] = queue.Queue() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + await self.async_costs.put(kwargs["standard_logging_object"]["response_cost"]) + + def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.sync_costs.put(kwargs["standard_logging_object"]["response_cost"]) + + +@pytest.mark.asyncio +async def test_atranscription_bills_the_transcript_when_the_model_ignores_stream( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"text": "hello world"}) + + recorder: Final = _SuccessCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/whisper-1", file=_one_second_wav(), client=client) + unstreamed_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + ignored_stream: Final = await litellm.atranscription( + model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client + ) + ignored_stream_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + await client.close() + + assert isinstance(ignored_stream, TranscriptionResponse) + assert ignored_stream.text == "hello world" + assert unstreamed_cost is not None + assert ignored_stream_cost == unstreamed_cost > 0 + + +def test_azure_transcription_bills_the_transcript_when_the_deployment_ignores_stream( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"text": "hello world"}) + + recorder: Final = _SuccessCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) + azure_params: Final = { + "client": client, + "api_key": "azure-test-key", + "api_base": "https://example.openai.azure.com", + "api_version": litellm.AZURE_DEFAULT_API_VERSION, + } + litellm.transcription(model="azure/whisper-1", file=_one_second_wav(), **azure_params) + unstreamed_cost: Final = recorder.sync_costs.get(timeout=5) + ignored_stream: Final = litellm.transcription( + model="azure/whisper-1", file=_one_second_wav(), stream=True, **azure_params + ) + ignored_stream_cost: Final = recorder.sync_costs.get(timeout=5) + client.close() + + assert isinstance(ignored_stream, TranscriptionResponse) + assert ignored_stream.text == "hello world" + assert unstreamed_cost is not None + assert ignored_stream_cost == unstreamed_cost > 0 + + +def _duration_priced_transcript(streamed: bool) -> httpx.Response: + usage: Final = {"type": "duration", "seconds": 2} + if not streamed: + return httpx.Response(200, json={"text": "hello world", "usage": usage}) + events: Final = ( + {"type": "transcript.text.delta", "delta": "hello world"}, + {"type": "transcript.text.done", "text": "hello world", "usage": usage}, + ) + return httpx.Response( + 200, + content="".join(f"data: {json.dumps(event)}\n\n" for event in events).encode(), + headers={"content-type": "text/event-stream"}, + ) + + +def _asks_for_stream(request_body: bytes) -> bool: + return b'name="stream"\r\n\r\ntrue' in request_body + + +@pytest.mark.asyncio +async def test_atranscription_stream_bills_the_provider_duration_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + recorder: Final = _SuccessCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + unstreamed_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + events: Final = [event.type async for event in stream] + streamed_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + await client.close() + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert unstreamed_cost is not None + assert streamed_cost == unstreamed_cost > 0 + + +def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(request.read())) + + recorder: Final = _SuccessCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) + litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + unstreamed_cost: Final = recorder.sync_costs.get(timeout=5) + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + events: Final = [event.type for event in stream] + streamed_cost: Final = recorder.sync_costs.get(timeout=5) + client.close() + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert unstreamed_cost is not None + assert streamed_cost == unstreamed_cost > 0 From e82a7650668038d9eb5563b669b07ca45d0990ba Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:23:12 -0700 Subject: [PATCH 38/45] fix(transcription): log streamed transcription usage from the provider response The stream logging path built the spend payload from the response after its duration usage was transformed away, so whisper-1 with stream=true and streamed gpt-transcribe logged an empty usage object. Bill and log from the provider's TranscriptionResponse in both the sync and async paths. --- litellm/litellm_core_utils/litellm_logging.py | 20 ++--- .../transcriptions/test_gpt_transcribe.py | 74 ++++++++++++------- 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index bfebc9b7667..a61aa077798 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2857,10 +2857,6 @@ class Logging(LiteLLMLoggingBaseClass): setattr(result, "usage", transformed_usage) return result - @staticmethod - def _stream_cost_source(received_result: object, assembled_response: object) -> object: - return received_result if isinstance(received_result, TranscriptionResponse) else assembled_response - def _with_client_facing_stream_model( self, response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, @@ -3233,7 +3229,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit) if not self.should_run_logging(event_type="sync_success"): # prevent double logging return - received_result: Final = result + provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None start_time, end_time, result, request_logging_payload = self._prepare_success_logging( start_time=start_time, end_time=end_time, @@ -3269,7 +3265,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["complete_streaming_response"] = complete_streaming_response self._surface_response_headers_from_result(complete_streaming_response) self.model_call_details["response_cost"] = self.response_cost_calculator( - result=self._stream_cost_source(received_result, complete_streaming_response) + result=complete_streaming_response if provider_transcription is None else provider_transcription ) self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) @@ -3280,7 +3276,9 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD if not for_async_request: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( - logged_streaming_response, start_time, end_time + logged_streaming_response if provider_transcription is None else provider_transcription, + start_time, + end_time, ) standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get( "standard_logging_object" @@ -3807,7 +3805,7 @@ class Logging(LiteLLMLoggingBaseClass): kwargs=self.model_call_details, messages=self.model_call_details.get("messages") ) ) - received_result: Final = result + provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None start_time, end_time, result = self._success_handler_helper_fn( start_time=start_time, end_time=end_time, @@ -3849,7 +3847,7 @@ class Logging(LiteLLMLoggingBaseClass): get_base_model_from_metadata(model_call_details=self.model_call_details) # base_model defaults to None if not set on model_info self.model_call_details["response_cost"] = self.response_cost_calculator( - result=self._stream_cost_source(received_result, complete_streaming_response) + result=complete_streaming_response if provider_transcription is None else provider_transcription ) verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"]) @@ -3878,7 +3876,9 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD try: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( - logged_streaming_response, start_time, end_time + logged_streaming_response if provider_transcription is None else provider_transcription, + start_time, + end_time, ) except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release) verbose_logger.exception( diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 10c145ba6df..35aec449bfc 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -3,7 +3,7 @@ import json import wave from collections.abc import Iterator from datetime import datetime -from typing import Final +from typing import Final, NamedTuple from unittest.mock import AsyncMock, MagicMock import httpx @@ -376,17 +376,27 @@ def _one_second_wav() -> io.BytesIO: return audio_file -class _SuccessCostRecorder(CustomLogger): +class _LoggedSpend(NamedTuple): + cost: float | None + usage: dict[str, object] | None + + +def _logged_spend(kwargs: dict[str, object]) -> _LoggedSpend: + payload: Final = kwargs["standard_logging_object"] + return _LoggedSpend(payload["response_cost"], payload["metadata"]["usage_object"]) + + +class _SuccessSpendRecorder(CustomLogger): def __init__(self) -> None: super().__init__() - self.async_costs: Final[asyncio.Queue[float | None]] = asyncio.Queue() - self.sync_costs: Final[queue.Queue[float | None]] = queue.Queue() + self.async_spends: Final[asyncio.Queue[_LoggedSpend]] = asyncio.Queue() + self.sync_spends: Final[queue.Queue[_LoggedSpend]] = queue.Queue() async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: - await self.async_costs.put(kwargs["standard_logging_object"]["response_cost"]) + await self.async_spends.put(_logged_spend(kwargs)) def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: - self.sync_costs.put(kwargs["standard_logging_object"]["response_cost"]) + self.sync_spends.put(_logged_spend(kwargs)) @pytest.mark.asyncio @@ -394,9 +404,9 @@ async def test_atranscription_bills_the_transcript_when_the_model_ignores_stream local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: async def send_response(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json={"text": "hello world"}) + return _duration_priced_transcript(streamed=False) - recorder: Final = _SuccessCostRecorder() + recorder: Final = _SuccessSpendRecorder() monkeypatch.setattr(litellm, "callbacks", [recorder]) client: Final = AsyncOpenAI( api_key="sk-test", @@ -404,26 +414,28 @@ async def test_atranscription_bills_the_transcript_when_the_model_ignores_stream http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), ) await litellm.atranscription(model="openai/whisper-1", file=_one_second_wav(), client=client) - unstreamed_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) ignored_stream: Final = await litellm.atranscription( model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client ) - ignored_stream_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + ignored_stream_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) await client.close() assert isinstance(ignored_stream, TranscriptionResponse) assert ignored_stream.text == "hello world" - assert unstreamed_cost is not None - assert ignored_stream_cost == unstreamed_cost > 0 + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert ignored_stream_spend == unstreamed_spend def test_azure_transcription_bills_the_transcript_when_the_deployment_ignores_stream( local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: def send_response(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, json={"text": "hello world"}) + return _duration_priced_transcript(streamed=False) - recorder: Final = _SuccessCostRecorder() + recorder: Final = _SuccessSpendRecorder() monkeypatch.setattr(litellm, "callbacks", [recorder]) client: Final = AzureOpenAI( api_key="azure-test-key", @@ -438,17 +450,19 @@ def test_azure_transcription_bills_the_transcript_when_the_deployment_ignores_st "api_version": litellm.AZURE_DEFAULT_API_VERSION, } litellm.transcription(model="azure/whisper-1", file=_one_second_wav(), **azure_params) - unstreamed_cost: Final = recorder.sync_costs.get(timeout=5) + unstreamed_spend: Final = recorder.sync_spends.get(timeout=5) ignored_stream: Final = litellm.transcription( model="azure/whisper-1", file=_one_second_wav(), stream=True, **azure_params ) - ignored_stream_cost: Final = recorder.sync_costs.get(timeout=5) + ignored_stream_spend: Final = recorder.sync_spends.get(timeout=5) client.close() assert isinstance(ignored_stream, TranscriptionResponse) assert ignored_stream.text == "hello world" - assert unstreamed_cost is not None - assert ignored_stream_cost == unstreamed_cost > 0 + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert ignored_stream_spend == unstreamed_spend def _duration_priced_transcript(streamed: bool) -> httpx.Response: @@ -477,7 +491,7 @@ async def test_atranscription_stream_bills_the_provider_duration_like_the_unstre async def send_response(request: httpx.Request) -> httpx.Response: return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) - recorder: Final = _SuccessCostRecorder() + recorder: Final = _SuccessSpendRecorder() monkeypatch.setattr(litellm, "callbacks", [recorder]) client: Final = AsyncOpenAI( api_key="sk-test", @@ -485,17 +499,19 @@ async def test_atranscription_stream_bills_the_provider_duration_like_the_unstre http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), ) await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) - unstreamed_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) stream: Final = await litellm.atranscription( model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client ) events: Final = [event.type async for event in stream] - streamed_cost: Final = await asyncio.wait_for(recorder.async_costs.get(), timeout=5) + streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) await client.close() assert events == ["transcript.text.delta", "transcript.text.done"] - assert unstreamed_cost is not None - assert streamed_cost == unstreamed_cost > 0 + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert streamed_spend == unstreamed_spend def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call( @@ -504,7 +520,7 @@ def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_ca def send_response(request: httpx.Request) -> httpx.Response: return _duration_priced_transcript(streamed=_asks_for_stream(request.read())) - recorder: Final = _SuccessCostRecorder() + recorder: Final = _SuccessSpendRecorder() monkeypatch.setattr(litellm, "callbacks", [recorder]) client: Final = OpenAI( api_key="sk-test", @@ -512,14 +528,16 @@ def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_ca http_client=httpx.Client(transport=httpx.MockTransport(send_response)), ) litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) - unstreamed_cost: Final = recorder.sync_costs.get(timeout=5) + unstreamed_spend: Final = recorder.sync_spends.get(timeout=5) stream: Final = litellm.transcription( model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client ) events: Final = [event.type for event in stream] - streamed_cost: Final = recorder.sync_costs.get(timeout=5) + streamed_spend: Final = recorder.sync_spends.get(timeout=5) client.close() assert events == ["transcript.text.delta", "transcript.text.done"] - assert unstreamed_cost is not None - assert streamed_cost == unstreamed_cost > 0 + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert streamed_spend == unstreamed_spend From 64fcaee7ab9b426d1f40c685c1813fd83a2c5830 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 06:45:43 -0700 Subject: [PATCH 39/45] fix(realtime): keep main's routing, protocol and transport behavior around the new translation routes - Send OpenAI realtime and translation client_secrets and calls through litellm's httpx client again, so ssl_verify and custom CA settings apply and upstream errors keep main's shape - Use the input transcription model as the session model only for transcription sessions - Drop the new /openai/v1 translation aliases and record each lazy feature's own routes, so the /openai pass-through and the MCP routes resolve as on main - Let a configured Azure realtime_protocol win for every model, with GA as the default only when none is set - Return x-litellm headers on streamed transcriptions and mark the request done - Refuse translation sessions before connecting upstream when realtime guardrails apply or the provider is not OpenAI or Azure - Accept json and text for gpt-transcribe, and drop other formats under drop_params instead of failing - Bill translation sessions on output audio seconds only, at the single published rate - Document the 201 SDP answer on the translation calls routes and the transcription 200 response --- .../audio_utils/transcription_streaming.py | 4 +- .../litellm_core_utils/realtime_streaming.py | 82 +- litellm/llms/azure/realtime/handler.py | 12 +- litellm/llms/custom_httpx/llm_http_handler.py | 164 +-- litellm/main.py | 27 +- ...odel_prices_and_context_window_backup.json | 5 +- litellm/proxy/_lazy_features.py | 22 +- litellm/proxy/_lazy_openapi_snapshot.json | 1004 +---------------- litellm/proxy/proxy_server.py | 81 +- litellm/proxy/realtime_endpoints/endpoints.py | 24 +- litellm/realtime_api/main.py | 98 +- model_prices_and_context_window.json | 5 +- .../llms/openai/realtime/test_translation.py | 99 -- .../transcriptions/test_gpt_transcribe.py | 79 +- .../proxy/proxy_server/test_routes_audio.py | 64 +- .../test_realtime_webrtc_endpoints.py | 31 +- tests/unit/proxy/test__lazy_features.py | 44 + tests/unit/realtime_api/test_main.py | 410 ++++++- tests/unit/test_cost_calculator.py | 121 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 254 +---- 20 files changed, 992 insertions(+), 1638 deletions(-) diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 719909d66ea..0dd18fe581b 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -2,7 +2,7 @@ from __future__ import annotations import datetime import traceback -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping from typing import Final, Protocol from openai import AsyncStream, Stream @@ -94,6 +94,7 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): start_time: datetime.datetime, ) -> None: self.__dict__.update(stream.__dict__) + self._hidden_params: Mapping[str, object] = {} self._logging_obj = logging_obj self._start_time = start_time self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) @@ -144,6 +145,7 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): start_time: datetime.datetime, ) -> None: self.__dict__.update(stream.__dict__) + self._hidden_params: Mapping[str, object] = {} self._logging_obj = logging_obj self._start_time = start_time self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 8b866666c6a..851fbdd51fb 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -249,6 +249,45 @@ DefaultLoggedRealTimeEventTypes: Final = [ ] +TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE: Final = "Translation sessions cannot enforce configured realtime guardrails" + + +def _guardrails_run_for_event_hooks( + request_data: Mapping[str, object], + event_hooks: Sequence["GuardrailEventHooks"], +) -> bool: + from litellm.integrations.custom_guardrail import CustomGuardrail + + return any( + isinstance(cb, CustomGuardrail) + and any(cb.should_run_guardrail(data={**request_data, "stream": True}, event_type=et) for et in event_hooks) + for cb in litellm.callbacks + ) + + +def realtime_guardrails_apply(request_data: Mapping[str, object]) -> bool: + from litellm.types.guardrails import GuardrailEventHooks + + if any( + isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket + for bucket in (request_data.get("metadata"), request_data.get("litellm_metadata")) + ): + from litellm.proxy.utils import pipeline_managed_guardrail_names + + if pipeline_managed_guardrail_names(request_data, "pre_call") or pipeline_managed_guardrail_names( + request_data, "post_call" + ): + return True + return _guardrails_run_for_event_hooks( + request_data, + ( + GuardrailEventHooks.realtime_input_transcription, + GuardrailEventHooks.pre_call, + GuardrailEventHooks.post_call, + ), + ) + + class RealTimeStreaming: def __init__( self, @@ -1170,41 +1209,10 @@ class RealTimeStreaming: self, event_hooks: Sequence["GuardrailEventHooks"], ) -> bool: - """Return True if any callback would run for one of ``event_hooks``.""" - from litellm.integrations.custom_guardrail import CustomGuardrail - - return any( - isinstance(cb, CustomGuardrail) - and any( - cb.should_run_guardrail( - data={**self.request_data, "stream": True}, - event_type=et, - ) - for et in event_hooks - ) - for cb in litellm.callbacks - ) + return _guardrails_run_for_event_hooks(self.request_data, event_hooks) def _has_realtime_guardrails(self) -> bool: - from litellm.types.guardrails import GuardrailEventHooks - - if any( - isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket - for bucket in (self.request_data.get("metadata"), self.request_data.get("litellm_metadata")) - ): - from litellm.proxy.utils import pipeline_managed_guardrail_names - - if pipeline_managed_guardrail_names(self.request_data, "pre_call") or pipeline_managed_guardrail_names( - self.request_data, "post_call" - ): - return True - return self._has_realtime_guardrails_for_event_hooks( - [ - GuardrailEventHooks.realtime_input_transcription, - GuardrailEventHooks.pre_call, - GuardrailEventHooks.post_call, - ] - ) + return realtime_guardrails_apply(self.request_data) def _has_audio_transcription_guardrails(self) -> bool: """Return True when a guardrail is configured for the audio/VAD transcript path. @@ -1976,13 +1984,9 @@ class RealTimeStreaming: async def bidirectional_forward(self) -> None: if self._is_translation_session and self._has_realtime_guardrails(): await self.websocket.send_text( - realtime_error_event( - "Translation sessions cannot enforce configured realtime guardrails", error_type="guardrail_error" - ) - ) - await self.websocket.close( - code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" + realtime_error_event(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, error_type="guardrail_error") ) + await self.websocket.close(code=1008, reason=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) return forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages()) client_task: Final = asyncio.create_task(self.client_ack_messages()) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index eb4e02f863b..4c243d12781 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -71,14 +71,14 @@ def azure_realtime_protocol_for_client( query_params: RealtimeQueryParams | None, websocket: ScopedWebSocket, ) -> str: - if azure_realtime_requires_ga(model): - if isinstance(configured_protocol, str) and configured_protocol.upper() not in ("GA", "V1"): - raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") - return "GA" - if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": - return "GA" if isinstance(configured_protocol, str) and configured_protocol: return configured_protocol + if ( + realtime_mode == "translation" + or (query_params or {}).get("intent") == "transcription" + or azure_realtime_requires_ga(model) + ): + return "GA" return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 46b1901895a..d0be5c5ae05 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -27,9 +27,7 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse import httpx from httpx import USE_CLIENT_DEFAULT from httpx._types import FileContent -from openai import AsyncOpenAI from openai.types.file_deleted import FileDeleted -from openai.types.realtime import RealtimeSessionCreateRequestParam from pydantic import TypeAdapter from typing_extensions import ReadOnly @@ -6849,9 +6847,8 @@ class BaseLLMHTTPHandler: provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, extra_headers: Mapping[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, - use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/client_secrets to upstream provider. @@ -6859,30 +6856,6 @@ class BaseLLMHTTPHandler: Uses provider_config (BaseRealtimeHTTPConfig) for URL construction and header auth when available; falls back to the legacy OpenAI-style defaults. """ - if use_openai_sdk: - openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) - logging_obj.pre_call( - input=request_data, - api_key="", - additional_args={ - "complete_input_dict": request_data, - "api_base": normalized_api_base, - }, - ) - try: - configured_client: Final = openai_client.with_options( - timeout=timeout, - set_default_headers={key: str(value) for key, value in (extra_headers or {}).items()}, - ) - raw_response: Final = await configured_client.post( - "/realtime/client_secrets", - cast_to=httpx.Response, - body=request_data, - ) - return self._decoded_realtime_sdk_response(raw_response) - finally: - if owns_client: - await openai_client.close() return await self._async_realtime_session_post( endpoint="client_secrets", api_base=api_base, @@ -6907,7 +6880,7 @@ class BaseLLMHTTPHandler: provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, extra_headers: Mapping[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: """Forward POST /v1/realtime/transcription_sessions to upstream provider.""" @@ -6935,36 +6908,9 @@ class BaseLLMHTTPHandler: provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, extra_headers: Mapping[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, - use_openai_sdk: bool = False, ) -> httpx.Response: - if use_openai_sdk: - openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) - logging_obj.pre_call( - input=request_data, - api_key="", - additional_args={ - "complete_input_dict": request_data, - "api_base": normalized_api_base, - }, - ) - try: - configured_client: Final = openai_client.with_options( - timeout=timeout, - set_default_headers=MappingProxyType( - {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} - ), - ) - raw_response: Final = await configured_client.post( - "/realtime/translations/client_secrets", - cast_to=httpx.Response, - body=request_data, - ) - return self._decoded_realtime_sdk_response(raw_response) - finally: - if owns_client: - await openai_client.close() return await self._async_realtime_session_post( endpoint="translation_client_secrets", api_base=api_base, @@ -6990,7 +6936,7 @@ class BaseLLMHTTPHandler: provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, extra_headers: Mapping[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: """ @@ -7059,92 +7005,6 @@ class BaseLLMHTTPHandler: ) raise - async def _async_realtime_calls_sdk( - self, - api_base: str, - openai_ephemeral_key: str, - sdp_text: str, - session_data: Mapping[str, object], - logging_obj: LiteLLMLoggingObj, - timeout: float | httpx.Timeout, - extra_headers: Mapping[str, object] | None, - client: object | None, - translation: bool, - ) -> httpx.Response: - openai_client, owns_client, normalized_api_base = self._realtime_sdk_client( - api_base, openai_ephemeral_key, client - ) - logging_obj.pre_call( - input="realtime_sdp_offer", - api_key="", - additional_args={ - "api_base": normalized_api_base, - "session": session_data, - }, - ) - try: - if translation: - configured_client: Final = openai_client.with_options( - timeout=timeout, - set_default_headers={ - "Content-Type": "application/sdp", - **MappingProxyType( - {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} - ), - }, - ) - translation_response: Final = await configured_client.post( - "/realtime/translations/calls", - cast_to=httpx.Response, - content=sdp_text.encode("utf-8"), - ) - return self._decoded_realtime_sdk_response(translation_response) - realtime_session_data: Final = cast( # cast-ok: endpoint validation produced an OpenAI realtime session - RealtimeSessionCreateRequestParam, - session_data, - ) - sdk_extra_headers: Final = MappingProxyType( - {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} - ) - raw_response: Final = await openai_client.realtime.calls.with_raw_response.create( - sdp=sdp_text, - session=realtime_session_data, - extra_headers=sdk_extra_headers, - timeout=timeout, - ) - return self._decoded_realtime_sdk_response(raw_response.http_response) - finally: - if owns_client: - await openai_client.close() - - @staticmethod - def _realtime_sdk_client(api_base: str, api_key: str, client: object | None) -> tuple[AsyncOpenAI, bool, str]: - trimmed_api_base: Final = api_base.rstrip("/") - normalized_api_base: Final = trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" - if isinstance(client, AsyncOpenAI): - return client, False, normalized_api_base - if isinstance(client, AsyncHTTPHandler): - return ( - AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0, http_client=client.client), - False, - normalized_api_base, - ) - return AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0), True, normalized_api_base - - @staticmethod - def _decoded_realtime_sdk_response(response: httpx.Response) -> httpx.Response: - headers: Final = { - key: value - for key, value in response.headers.items() - if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") - } - return httpx.Response( - status_code=response.status_code, - headers=headers, - content=response.content, - request=response.request, - ) - @staticmethod def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: if isinstance(client, AsyncHTTPHandler): @@ -7162,10 +7022,9 @@ class BaseLLMHTTPHandler: model: str | None = None, session_config: Mapping[str, object] | None = None, extra_headers: Mapping[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, translation: bool = False, - use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. @@ -7187,19 +7046,6 @@ class BaseLLMHTTPHandler: sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body - if use_openai_sdk: - return await self._async_realtime_calls_sdk( - api_base=api_base, - openai_ephemeral_key=openai_ephemeral_key, - sdp_text=sdp_text, - session_data=session_data, - logging_obj=logging_obj, - timeout=timeout, - extra_headers=extra_headers, - client=client, - translation=translation, - ) - async_httpx_client: Final = self._get_realtime_async_http_client(client) if provider_config is not None: diff --git a/litellm/main.py b/litellm/main.py index 6953e1d8cbb..b38a33fbd4f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8256,7 +8256,8 @@ def _validate_gpt_transcription_request( language: str | None, languages: Sequence[str] | None, response_format: str | None, -) -> None: + drop_params: object, +) -> str | None: model_cost_key: Final = next( (key for key in (f"{custom_llm_provider}/{model}", model) if key in litellm.model_cost), None ) @@ -8278,15 +8279,18 @@ def _validate_gpt_transcription_request( llm_provider=custom_llm_provider, ) if ( - provider_specific_entry is not None - and provider_specific_entry.get("transcription_json_only") == 1 - and response_format not in (None, "json") + provider_specific_entry is None + or provider_specific_entry.get("transcription_json_or_text_only") != 1 + or response_format in (None, "json", "text") ): - raise litellm.UnsupportedParamsError( - message=f"{model} only supports response_format='json'", - model=model, - llm_provider=custom_llm_provider, - ) + return response_format + if litellm.drop_params is True or normalize_drop_params(drop_params) is True: + return None + raise litellm.UnsupportedParamsError( + message=f"{model} only supports response_format 'json' or 'text'", + model=model, + llm_provider=custom_llm_provider, + ) @client @@ -8354,12 +8358,13 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key - _validate_gpt_transcription_request( + supported_response_format: Final = _validate_gpt_transcription_request( model=model, custom_llm_provider=custom_llm_provider, language=language, languages=languages, response_format=response_format, + drop_params=kwargs.get("drop_params"), ) optional_params: Final = get_optional_params_transcription( @@ -8368,7 +8373,7 @@ def transcription( languages=languages, keywords=keywords, prompt=prompt, - response_format=response_format, + response_format=supported_response_format, timestamp_granularities=timestamp_granularities, temperature=temperature, stream=stream, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 163d501ec01..7c5dfad023a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63357,7 +63357,7 @@ "litellm_provider": "openai", "mode": "audio_transcription", "provider_specific_entry": { - "transcription_json_only": 1, + "transcription_json_or_text_only": 1, "transcription_usage_duration": 1 }, "source": "https://developers.openai.com/api/docs/pricing", @@ -63412,7 +63412,6 @@ "supports_function_calling": true }, "gpt-realtime-translate": { - "input_cost_per_second": 0.000566666666667, "litellm_provider": "openai", "max_input_tokens": 16000, "max_output_tokens": 2000, @@ -71190,7 +71189,7 @@ "provider_specific_entry": { "realtime_ga_only": 1, "transcription_deployment_api": 1, - "transcription_json_only": 1, + "transcription_json_or_text_only": 1, "transcription_usage_duration": 1 }, "supports_native_streaming": true diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 0feb32592d8..cefb2d4646f 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -41,20 +41,6 @@ def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], return _register -def _include_router_prepend(attr_name: str = "router") -> Callable[["FastAPI", object], None]: - def _register(app: "FastAPI", module: object) -> None: - router: Final = app.router - route_count: Final = len(router.routes) - app.include_router(getattr(module, attr_name)) - new_routes: Final = router.routes[route_count:] - router.routes[:] = [ - *new_routes, - *router.routes[:route_count], - ] - - return _register - - def _mount_app(prefix: str, attr_name: str = "app") -> Callable[["FastAPI", object], None]: def _register(app: "FastAPI", module: object) -> None: app.mount(path=prefix, app=getattr(module, attr_name)) @@ -260,7 +246,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( name="realtime", module_path="litellm.proxy.realtime_endpoints.endpoints", path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"), - register_fn=_include_router_prepend(), ), LazyFeature( name="anthropic_passthrough", @@ -518,10 +503,13 @@ def _lazy_lock(app: "FastAPI", module_path: str) -> asyncio.Lock: def _register_feature(app: "FastAPI", feat: LazyFeature, module: object, features: tuple[LazyFeature, ...]) -> None: - before: Final = len(app.router.routes) + routes_before: Final = frozenset(id(route) for route in app.router.routes) feat.register_fn(app, module) lazy_routes: Final[Mapping[str, tuple[BaseRoute, ...]]] = MappingProxyType( - {**_lazy_routes(app), feat.module_path: tuple(app.router.routes[before:])} + { + **_lazy_routes(app), + feat.module_path: tuple(route for route in app.router.routes if id(route) not in routes_before), + } ) app.state.lazy_routes = lazy_routes # rebind-ok: the app owns the record of which routes each feature added app.router.routes[:] = _route_table( # rebind-ok: the app owns its route table diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 4f9ba0e6ed8..7e544cf8aa9 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -33219,501 +33219,6 @@ ] } }, - "/openai/v1/realtime/translations/calls": { - "post": { - "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", - "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post", - "requestBody": { - "content": { - "application/sdp": { - "schema": { - "type": "string" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": {} - }, - "application/sdp": { - "schema": { - "type": "string" - } - } - }, - "description": "WebRTC SDP answer" - } - }, - "summary": "Proxy Realtime Calls", - "tags": [ - "llm_passthrough" - ] - } - }, - "/openai/v1/realtime/translations/client_secrets": { - "post": { - "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", - "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post", - "requestBody": { - "content": { - "application/json": { - "example": { - "model": "gpt-realtime-translate", - "session": { - "model": "gpt-realtime-translate", - "type": "translation" - } - }, - "schema": { - "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", - "properties": { - "expires_after": { - "anyOf": [ - { - "description": "Expiration config for a client secret.", - "properties": { - "anchor": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "created_at", - "title": "Anchor" - }, - "seconds": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Seconds" - } - }, - "title": "RealtimeExpiresAfter", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "session": { - "anyOf": [ - { - "additionalProperties": true, - "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", - "properties": { - "audio": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "input": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "format": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Format" - }, - "noise_reduction": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Noise Reduction" - }, - "transcription": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "delay": { - "anyOf": [ - { - "enum": [ - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Delay" - }, - "keywords": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Keywords" - }, - "language": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Language" - }, - "languages": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Languages" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "prompt": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Prompt" - } - }, - "title": "RealtimeAudioTranscriptionConfig", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "turn_detection": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Turn Detection" - } - }, - "title": "RealtimeAudioInputConfig", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "output": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "format": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Format" - }, - "language": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Language" - }, - "voice": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Voice" - } - }, - "title": "RealtimeAudioOutputConfig", - "type": "object" - }, - { - "type": "null" - } - ] - } - }, - "title": "RealtimeSessionAudioConfig", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "include": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Include" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Instructions" - }, - "max_output_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Max Output Tokens" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "output_modalities": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Output Modalities" - }, - "prompt": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Prompt" - }, - "tool_choice": { - "anyOf": [ - {}, - { - "type": "null" - } - ], - "title": "Tool Choice" - }, - "tools": { - "anyOf": [ - { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Tools" - }, - "tracing": { - "anyOf": [ - {}, - { - "type": "null" - } - ], - "title": "Tracing" - }, - "truncation": { - "anyOf": [ - {}, - { - "type": "null" - } - ], - "title": "Truncation" - }, - "type": { - "anyOf": [ - { - "enum": [ - "realtime", - "transcription", - "translation" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Type" - } - }, - "title": "RealtimeSessionConfig", - "type": "object" - }, - { - "type": "null" - } - ] - } - }, - "title": "RealtimeClientSecretRequest", - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RealtimeClientSecretResponse" - } - } - }, - "description": "Successful Response" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Create Realtime Client Secret", - "tags": [ - "llm_passthrough" - ] - } - }, "/openai/v1/responses": { "post": { "description": "Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses\n\nSupports background mode with polling_via_cache for partial response retrieval.\nWhen background=true and polling_via_cache is enabled, returns a polling_id immediately\nand streams the response in the background, updating Redis cache.\n\n```bash\n# Normal request\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\"\n}'\n\n# Background request with polling\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer $LITELLM_MASTER_KEY\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\",\n \"background\": true\n}'\n```", @@ -53859,501 +53364,6 @@ ] } }, - "/openai/v1/realtime/translations/calls": { - "post": { - "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", - "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post_2", - "requestBody": { - "content": { - "application/sdp": { - "schema": { - "type": "string" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": {} - }, - "application/sdp": { - "schema": { - "type": "string" - } - } - }, - "description": "WebRTC SDP answer" - } - }, - "summary": "Proxy Realtime Calls", - "tags": [ - "realtime" - ] - } - }, - "/openai/v1/realtime/translations/client_secrets": { - "post": { - "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", - "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post_2", - "requestBody": { - "content": { - "application/json": { - "example": { - "model": "gpt-realtime-translate", - "session": { - "model": "gpt-realtime-translate", - "type": "translation" - } - }, - "schema": { - "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", - "properties": { - "expires_after": { - "anyOf": [ - { - "description": "Expiration config for a client secret.", - "properties": { - "anchor": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": "created_at", - "title": "Anchor" - }, - "seconds": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Seconds" - } - }, - "title": "RealtimeExpiresAfter", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "session": { - "anyOf": [ - { - "additionalProperties": true, - "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", - "properties": { - "audio": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "input": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "format": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Format" - }, - "noise_reduction": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Noise Reduction" - }, - "transcription": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "delay": { - "anyOf": [ - { - "enum": [ - "minimal", - "low", - "medium", - "high", - "xhigh" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Delay" - }, - "keywords": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Keywords" - }, - "language": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Language" - }, - "languages": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Languages" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "prompt": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Prompt" - } - }, - "title": "RealtimeAudioTranscriptionConfig", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "turn_detection": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Turn Detection" - } - }, - "title": "RealtimeAudioInputConfig", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "output": { - "anyOf": [ - { - "additionalProperties": true, - "properties": { - "format": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Format" - }, - "language": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Language" - }, - "voice": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Voice" - } - }, - "title": "RealtimeAudioOutputConfig", - "type": "object" - }, - { - "type": "null" - } - ] - } - }, - "title": "RealtimeSessionAudioConfig", - "type": "object" - }, - { - "type": "null" - } - ] - }, - "include": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Include" - }, - "instructions": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Instructions" - }, - "max_output_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Max Output Tokens" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Model" - }, - "output_modalities": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Output Modalities" - }, - "prompt": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "Prompt" - }, - "tool_choice": { - "anyOf": [ - {}, - { - "type": "null" - } - ], - "title": "Tool Choice" - }, - "tools": { - "anyOf": [ - { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "title": "Tools" - }, - "tracing": { - "anyOf": [ - {}, - { - "type": "null" - } - ], - "title": "Tracing" - }, - "truncation": { - "anyOf": [ - {}, - { - "type": "null" - } - ], - "title": "Truncation" - }, - "type": { - "anyOf": [ - { - "enum": [ - "realtime", - "transcription", - "translation" - ], - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Type" - } - }, - "title": "RealtimeSessionConfig", - "type": "object" - }, - { - "type": "null" - } - ] - } - }, - "title": "RealtimeClientSecretRequest", - "type": "object" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RealtimeClientSecretResponse" - } - } - }, - "description": "Successful Response" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Create Realtime Client Secret", - "tags": [ - "realtime" - ] - } - }, "/realtime/calls": { "post": { "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", @@ -54443,12 +53453,9 @@ "required": true }, "responses": { - "200": { + "201": { "content": { - "application/json": { - "schema": {} - }, - "application/sdp": { + "text/plain": { "schema": { "type": "string" } @@ -55012,12 +54019,9 @@ "required": true }, "responses": { - "200": { + "201": { "content": { - "application/json": { - "schema": {} - }, - "application/sdp": { + "text/plain": { "schema": { "type": "string" } diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c054ef65daf..53ce7652256 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -199,6 +199,7 @@ from litellm.types.utils import ( TextCompletionResponse, TokenCountMessagesFormat, TokenCountResponse, + TranscriptionResponse, echoed_cost_map_pricing_fields, is_server_derived_pricing_key, pricing_override_fields, @@ -13360,11 +13361,13 @@ def _form_fields_without_file(form_data: Mapping[str, object]) -> Mapping[str, o "/v1/audio/transcriptions", dependencies=[Depends(user_api_key_auth)], tags=["audio"], + responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}}, ) @router.post( "/audio/transcriptions", dependencies=[Depends(user_api_key_auth)], tags=["audio"], + responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}}, ) async def audio_transcriptions( request: Request, @@ -13452,9 +13455,42 @@ async def audio_transcriptions( finally: file_object.close() # close the file read in by io library - if data.get("stream") is True: - if not hasattr(response, "__aiter__"): - raise TypeError(f"Streaming transcription returned {type(response).__name__}, expected an async stream") + ### ALERTING ### + asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) + + ### RESPONSE HEADERS ### + hidden_params: Final = getattr(response, "_hidden_params", {}) or {} + model_id: Final = hidden_params.get("model_id", None) or "" + cache_key: Final = hidden_params.get("cache_key", None) or "" + api_base: Final = hidden_params.get("api_base", None) or "" + response_cost: Final = hidden_params.get("response_cost", None) or "" + response_call_id: Final = hidden_params.get("litellm_call_id", None) or litellm_call_id + additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {} + + custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + call_id=response_call_id, + request_data=data, + hidden_params=hidden_params, + **additional_headers, + ) + + # Call response headers hook (matches base_process_llm_request behavior) + callback_headers: Final = await proxy_logging_obj.post_call_response_headers_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=dict(request.headers), + ) + response_headers: Final = {**custom_headers, **(callback_headers or {})} + + if data.get("stream") is True and hasattr(response, "__aiter__"): stream_response: Final = cast(AsyncIterator[TranscriptionStreamEvent], response) async def transcription_event_stream( @@ -13473,45 +13509,10 @@ async def audio_transcriptions( return StreamingResponse( transcription_event_stream(stream_response), media_type="text/event-stream", + headers=response_headers, ) - ### ALERTING ### - asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) - - ### RESPONSE HEADERS ### - hidden_params: Final = getattr(response, "_hidden_params", {}) or {} - model_id: Final = hidden_params.get("model_id", None) or "" - cache_key: Final = hidden_params.get("cache_key", None) or "" - api_base: Final = hidden_params.get("api_base", None) or "" - response_cost: Final = hidden_params.get("response_cost", None) or "" - response_call_id: Final = hidden_params.get("litellm_call_id", None) or "" - additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {} - - fastapi_response.headers.update( - ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - model_id=model_id, - cache_key=cache_key, - api_base=api_base, - version=version, - response_cost=response_cost, - model_region=getattr(user_api_key_dict, "allowed_model_region", ""), - call_id=response_call_id, - request_data=data, - hidden_params=hidden_params, - **additional_headers, - ) - ) - - # Call response headers hook (matches base_process_llm_request behavior) - callback_headers: Final = await proxy_logging_obj.post_call_response_headers_hook( - data=data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=dict(request.headers), - ) - if callback_headers: - fastapi_response.headers.update(callback_headers) + fastapi_response.headers.update(response_headers) return response except Exception as e: diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index f86f99c8ff3..5471cfc09af 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi import status as http_status +from fastapi.responses import PlainTextResponse from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException, UserAPIKeyAuth @@ -59,9 +60,6 @@ _TRANSLATION_SECRET_BODY: Final = { _TRANSLATION_SDP_BODY: Final = { "requestBody": {"required": True, "content": {"application/sdp": {"schema": {"type": "string"}}}} } -_TRANSLATION_SDP_RESPONSES: Final[dict[int | str, dict[str, object]]] = { - 200: {"description": "WebRTC SDP answer", "content": {"application/sdp": {"schema": {"type": "string"}}}} -} def _enforce_non_billable_realtime_protocol_gate(general_settings: Mapping[str, object]) -> None: @@ -314,12 +312,6 @@ def _decode_realtime_token_payload( dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) -@router.post( - "/openai/v1/realtime/translations/client_secrets", - openapi_extra=_TRANSLATION_SECRET_BODY, - dependencies=[Depends(user_api_key_auth)], - tags=["realtime"], -) async def create_realtime_client_secret( request: Request, fastapi_response: Response, @@ -472,20 +464,18 @@ async def create_realtime_client_secret( ) @router.post( "/v1/realtime/translations/calls", + status_code=http_status.HTTP_201_CREATED, + response_class=PlainTextResponse, + response_description="WebRTC SDP answer", openapi_extra=_TRANSLATION_SDP_BODY, - responses=_TRANSLATION_SDP_RESPONSES, tags=["realtime"], ) @router.post( "/realtime/translations/calls", + status_code=http_status.HTTP_201_CREATED, + response_class=PlainTextResponse, + response_description="WebRTC SDP answer", openapi_extra=_TRANSLATION_SDP_BODY, - responses=_TRANSLATION_SDP_RESPONSES, - tags=["realtime"], -) -@router.post( - "/openai/v1/realtime/translations/calls", - openapi_extra=_TRANSLATION_SDP_BODY, - responses=_TRANSLATION_SDP_RESPONSES, tags=["realtime"], ) async def proxy_realtime_calls( diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index c5add19291d..20c24e03a76 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -8,7 +8,6 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast import httpx -from openai import AsyncOpenAI from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never @@ -42,7 +41,12 @@ from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor from ..litellm_core_utils.get_litellm_params import get_litellm_params from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from ..litellm_core_utils.realtime_streaming import client_websocket_scope +from ..litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason +from ..litellm_core_utils.realtime_streaming import ( + TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, + client_websocket_scope, + realtime_guardrails_apply, +) from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed @@ -111,6 +115,72 @@ def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]: return {**request_metadata, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})} +_HTTP_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) +_WEBSOCKET_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, *AZURE_OPENAI_AUDIO_PROVIDERS}) + + +@dataclass(frozen=True, slots=True) +class _TranslationProviderUnsupported: + provider: str + supported_providers: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class _TranslationGuardrailsUnsupported: + pass + + +_TranslationRejection = _TranslationProviderUnsupported | _TranslationGuardrailsUnsupported + + +def _translation_rejection( + custom_llm_provider: str, supported_providers: frozenset[str], kwargs: Mapping[str, object] +) -> _TranslationRejection | None: + if custom_llm_provider not in supported_providers: + return _TranslationProviderUnsupported(provider=custom_llm_provider, supported_providers=supported_providers) + if realtime_guardrails_apply({"litellm_metadata": _build_litellm_metadata(kwargs)}): + return _TranslationGuardrailsUnsupported() + return None + + +def _translation_rejection_message(rejection: _TranslationRejection) -> str: + match rejection: + case _TranslationProviderUnsupported(provider=provider, supported_providers=supported_providers): + return ( + f"Realtime translation supports only the {', '.join(sorted(supported_providers))} providers, " + f"got {provider}" + ) + case _TranslationGuardrailsUnsupported(): + return TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE + case _: + assert_never(rejection) + + +def _translation_rejection_error_type(rejection: _TranslationRejection) -> str: + match rejection: + case _TranslationProviderUnsupported(): + return "invalid_request_error" + case _TranslationGuardrailsUnsupported(): + return "guardrail_error" + case _: + assert_never(rejection) + + +def _raise_if_http_translation_rejected(custom_llm_provider: str, model: str, kwargs: Mapping[str, object]) -> None: + rejection: Final = _translation_rejection(custom_llm_provider, _HTTP_TRANSLATION_PROVIDERS, kwargs) + if rejection is None: + return + raise litellm.BadRequestError( + message=_translation_rejection_message(rejection), model=model, llm_provider=custom_llm_provider + ) + + +async def _refuse_translation_session(websocket: "WebSocket", rejection: _TranslationRejection) -> None: + message: Final = _translation_rejection_message(rejection) + await websocket.send_text(realtime_error_event(message, error_type=_translation_rejection_error_type(rejection))) + await websocket.close(code=1008, reason=websocket_close_reason(message, fallback="Translation session refused")) + + def _get_realtime_http_provider_config( custom_llm_provider: str, dynamic_api_base: str | None, @@ -189,6 +259,7 @@ async def acreate_realtime_client_secret( transcription_model: Final = ( req.session.audio.input.transcription.model if req.session is not None + and req.session.type == "transcription" and req.session.audio is not None and req.session.audio.input is not None and req.session.audio.input.transcription is not None @@ -257,7 +328,6 @@ async def acreate_realtime_client_secret( extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, - use_openai_sdk=custom_llm_provider == "openai", ) @@ -293,6 +363,7 @@ async def acreate_realtime_translation_client_secret( api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) + _raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs) provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( custom_llm_provider=custom_llm_provider, dynamic_api_base=dynamic_api_base, @@ -332,11 +403,8 @@ async def acreate_realtime_translation_client_secret( provider_config=provider_config, model=model_name, extra_headers=resolved_extra_headers, - client=client - if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler, AsyncOpenAI)) - else None, + client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None, api_version=litellm_params.api_version, - use_openai_sdk=custom_llm_provider == "openai", ) @@ -472,7 +540,6 @@ async def arealtime_calls( extra_headers=kwargs.get("extra_headers"), client=kwargs.get("client"), api_version=litellm_params.api_version, - use_openai_sdk=custom_llm_provider == "openai", ) @@ -497,6 +564,7 @@ async def arealtime_translation_calls( api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) + _raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs) provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( custom_llm_provider=custom_llm_provider, dynamic_api_base=dynamic_api_base, @@ -531,12 +599,9 @@ async def arealtime_translation_calls( model=model_name, session_config=session_config, extra_headers=as_str_mapping(kwargs.get("extra_headers")), - client=client - if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler, AsyncOpenAI)) - else None, + client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None, api_version=litellm_params.api_version, translation=True, - use_openai_sdk=custom_llm_provider == "openai", ) @@ -714,6 +779,15 @@ async def _arealtime( custom_llm_provider=_custom_llm_provider, ) + translation_rejection: Final = ( + _translation_rejection(_custom_llm_provider, _WEBSOCKET_TRANSLATION_PROVIDERS, kwargs) + if realtime_mode == "translation" + else None + ) + if translation_rejection is not None: + await _refuse_translation_session(websocket, translation_rejection) + return + if live_session_start is not None: await _alive_session( model=model, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 163d501ec01..7c5dfad023a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63357,7 +63357,7 @@ "litellm_provider": "openai", "mode": "audio_transcription", "provider_specific_entry": { - "transcription_json_only": 1, + "transcription_json_or_text_only": 1, "transcription_usage_duration": 1 }, "source": "https://developers.openai.com/api/docs/pricing", @@ -63412,7 +63412,6 @@ "supports_function_calling": true }, "gpt-realtime-translate": { - "input_cost_per_second": 0.000566666666667, "litellm_provider": "openai", "max_input_tokens": 16000, "max_output_tokens": 2000, @@ -71190,7 +71189,7 @@ "provider_specific_entry": { "realtime_ga_only": 1, "transcription_deployment_api": 1, - "transcription_json_only": 1, + "transcription_json_or_text_only": 1, "transcription_usage_duration": 1 }, "supports_native_streaming": true diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py index 97e9f822253..525782704c1 100644 --- a/tests/unit/llms/openai/realtime/test_translation.py +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -1,11 +1,8 @@ -import gzip -import json from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest -from openai import AsyncOpenAI from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -186,99 +183,3 @@ async def test_translation_calls_use_translation_session_and_path(): assert call["url"] == "https://api.openai.com/v1/realtime/translations/calls" assert call["headers"]["Content-Type"] == "application/sdp" assert call["content"] == "v=0\r\n" - - -@pytest.mark.asyncio -async def test_standard_client_secret_uses_openai_sdk_resource(): - async def send_response(request: httpx.Request) -> httpx.Response: - assert request.url.path == "/v1/realtime/client_secrets" - return httpx.Response( - 200, - content=gzip.compress(b'{"value":"ek_test"}'), - headers={"content-encoding": "gzip"}, - ) - - http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) - openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) - handler = BaseLLMHTTPHandler() - logging_obj = MagicMock() - - response = await handler.async_realtime_client_secret_handler( - api_base="https://example.com", - api_key="sk-test", - request_data={"session": {"type": "realtime", "model": "gpt-realtime-2.1"}}, - logging_obj=logging_obj, - timeout=10, - client=openai_client, - use_openai_sdk=True, - ) - await openai_client.close() - - assert response.status_code == 200 - assert response.json() == {"value": "ek_test"} - assert "content-encoding" not in response.headers - - -@pytest.mark.asyncio -async def test_translation_client_secret_uses_openai_sdk_custom_post(): - async def send_response(request: httpx.Request) -> httpx.Response: - assert request.url.path == "/v1/realtime/translations/client_secrets" - assert json.loads(request.content)["session"]["audio"]["output"]["language"] == "es" - return httpx.Response(200, content=gzip.compress(b'{"value":"ek_translation"}'), headers={"content-encoding": "gzip"}) - - http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) - openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) - handler = BaseLLMHTTPHandler() - - response = await handler.async_realtime_translation_client_secret_handler( - api_base="https://example.com", - api_key="sk-test", - request_data={ - "session": { - "model": "gpt-realtime-translate", - "audio": {"output": {"language": "es"}}, - } - }, - logging_obj=MagicMock(), - timeout=10, - client=openai_client, - use_openai_sdk=True, - ) - await openai_client.close() - - assert response.status_code == 200 - assert response.json() == {"value": "ek_translation"} - assert "content-encoding" not in response.headers - assert int(response.headers["content-length"]) == len(response.content) - - -@pytest.mark.asyncio -async def test_translation_calls_use_openai_sdk_custom_post(): - async def send_response(request: httpx.Request) -> httpx.Response: - assert request.url.path == "/v1/realtime/translations/calls" - body = await request.aread() - assert request.headers["content-type"] == "application/sdp" - assert body == b"v=0\r\n" - return httpx.Response(201, content=gzip.compress(b"v=0\r\n"), headers={"content-encoding": "gzip"}) - - http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) - openai_client = AsyncOpenAI(api_key="ek_test", base_url="https://example.com/v1", http_client=http_client) - handler = BaseLLMHTTPHandler() - - response = await handler.async_realtime_calls_handler( - api_base="https://example.com", - openai_ephemeral_key="ek_test", - sdp_body=b"v=0\r\n", - logging_obj=MagicMock(), - timeout=10, - model="gpt-realtime-translate", - client=openai_client, - translation=True, - use_openai_sdk=True, - ) - await openai_client.close() - - assert response.status_code == 201 - assert response.text == "v=0\r\n" - assert "content-encoding" not in response.headers - assert int(response.headers["content-length"]) == len(response.content) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 35aec449bfc..df5f96d6581 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -1,7 +1,7 @@ import io import json import wave -from collections.abc import Iterator +from collections.abc import Callable, Iterator from datetime import datetime from typing import Final, NamedTuple from unittest.mock import AsyncMock, MagicMock @@ -266,7 +266,7 @@ def test_gpt_transcribe_rejects_conflicting_language_inputs(): def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None: audio_file = io.BytesIO(b"audio") audio_file.name = "sample.wav" - with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format='json'"): + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"): litellm.transcription( model=model, file=audio_file, @@ -275,6 +275,81 @@ def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: N ) +@pytest.mark.parametrize("drop_globally", (True, False)) +def test_gpt_transcribe_drops_whisper_response_formats_under_drop_params( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, drop_globally: bool +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}}) + + monkeypatch.setattr(litellm, "drop_params", drop_globally) + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + response: Final = litellm.transcription( + model="gpt-transcribe", + file=audio_file, + response_format="verbose_json", + client=client, + **({} if drop_globally else {"drop_params": True}), + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b"verbose_json" not in sent_bodies[0] + + +def _openai_transcription_client(transport: httpx.MockTransport) -> OpenAI: + return OpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=httpx.Client(transport=transport)) + + +def _azure_transcription_client(transport: httpx.MockTransport) -> AzureOpenAI: + return AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=httpx.Client(transport=transport), + ) + + +@pytest.mark.parametrize( + ("model", "make_client"), + (("gpt-transcribe", _openai_transcription_client), ("azure/gpt-transcribe", _azure_transcription_client)), +) +def test_gpt_transcribe_sends_text_response_format_to_the_provider( + local_model_cost_map: None, model: str, make_client: Callable[[httpx.MockTransport], OpenAI | AzureOpenAI] +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, text="hello", headers={"content-type": "text/plain"}) + + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with make_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model=model, + file=audio_file, + response_format="text", + api_key="sk-test", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="response_format"\r\n\r\ntext\r\n' in sent_bodies[0] + + def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: None) -> None: audio_file = io.BytesIO(b"audio") audio_file.name = "sample.wav" diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index 03cfe3fa67e..7f186737f27 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -10,7 +10,7 @@ Pins (PR2): from __future__ import annotations import io -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest from fastapi.testclient import TestClient +from openai import AsyncOpenAI import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -350,9 +351,58 @@ def test_streaming_transcription_rejects_applicable_output_guardrails( data={"model": "gpt-transcribe", "stream": "true"}, ) - assert response.status_code == expected_status - if expected_status == 400: - assert "stream=false" in response.json()["error"]["message"] - assert "hello world" not in response.text - else: - assert '"text":"hello world"' in response.text + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert '"text":"hello world"' in response.text + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_returns_the_litellm_call_and_model_headers( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + content=b'data: {"type":"transcript.text.done","text":"hello world"}\n\n', + headers={"content-type": "text/event-stream"}, + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription( + model="openai/gpt-transcribe", + file=data["file"], + stream=True, + client=upstream, + litellm_call_id=data["litellm_call_id"], + model_info={"id": "transcription-deployment"}, + ) + + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + headers={"x-litellm-call-id": "streamed-transcription-call"}, + ) + + assert response.status_code == 200 + assert '"text":"hello world"' in response.text + assert response.headers["x-litellm-call-id"] == "streamed-transcription-call" + assert response.headers["x-litellm-model-id"] == "transcription-deployment" + assert "x-litellm-response-cost" not in response.headers + proxy_server.proxy_logging_obj.update_request_status.assert_called_once_with( + litellm_call_id="streamed-transcription-call", status="success" + ) diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index f9f75c39db4..387e09a7a70 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -14,7 +14,9 @@ import httpx import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient +from starlette.routing import Match +from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import ConfigGeneralSettings, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -1161,12 +1163,38 @@ async def test_translation_client_secret_binds_authorized_nested_transcription_m assert session_data["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" +@pytest.mark.parametrize( + "path", + ( + "/openai/v1/realtime/client_secrets", + "/openai/v1/realtime/calls", + "/openai/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/calls", + ), +) +def test_openai_prefixed_realtime_http_paths_stay_with_the_openai_pass_through( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import openai_proxy_route + + monkeypatch.setenv("LITELLM_DISABLE_LAZY_ROUTES", "true") + features = tuple(feature for feature in LAZY_FEATURES if feature.name in ("llm_passthrough", "realtime")) + app = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + scope = {"type": "http", "method": "POST", "path": path, "root_path": "", "headers": [], "query_string": b""} + + with TestClient(app): + resolved = next(route for route in app.router.routes if route.matches(scope)[0] is Match.FULL) + + assert getattr(resolved, "endpoint", None) is openai_proxy_route + + @pytest.mark.parametrize( "path", [ "/v1/realtime/translations/client_secrets", "/realtime/translations/client_secrets", - "/openai/v1/realtime/translations/client_secrets", ], ) def test_translation_client_secret_aliases_bind_token_family( @@ -1232,7 +1260,6 @@ def test_translation_client_secret_aliases_bind_token_family( [ "/v1/realtime/translations/calls", "/realtime/translations/calls", - "/openai/v1/realtime/translations/calls", ], ) def test_translation_calls_aliases_route_translation_session( diff --git a/tests/unit/proxy/test__lazy_features.py b/tests/unit/proxy/test__lazy_features.py index adc4de8c8fa..a4dae82a15a 100644 --- a/tests/unit/proxy/test__lazy_features.py +++ b/tests/unit/proxy/test__lazy_features.py @@ -1,6 +1,7 @@ import sys from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager +from dataclasses import replace from types import ModuleType from typing import Final @@ -10,6 +11,7 @@ from fastapi.testclient import TestClient from pydantic import BaseModel from litellm.proxy._lazy_features import ( + LAZY_FEATURES, LazyFeature, LazyFeatureMiddleware, attach_lazy_features, @@ -423,3 +425,45 @@ def test_root_authorization_server_metadata_is_the_gateways_not_the_byok_connect assert root.json()["token_endpoint"] == "https://gateway.example.com/token" assert connector.status_code == 200 assert connector.json()["token_endpoint"] == "https://gateway.example.com/v1/mcp/oauth/token" + + +@pytest.mark.parametrize( + "load_order", + (("/realtime/unrouted", "/mcp/unrouted"), ("/mcp/unrouted", "/realtime/unrouted")), + ids=("realtime_first", "realtime_last"), +) +def test_realtime_and_mcp_features_keep_every_route_once_in_either_load_order( + monkeypatch: pytest.MonkeyPatch, load_order: tuple[str, str] +) -> None: + monkeypatch.delenv(FLAG, raising=False) + + async def mcp_mount() -> Mapping[str, str]: + return {"route": "mcp_mount"} + + mounted_app: Final = FastAPI() + mounted_app.add_api_route("/{path:path}", mcp_mount, methods=["GET"]) + mcp_module: Final = ModuleType("tests.unit.proxy.lazy_fixture_mcp_app") + mcp_module.app = mounted_app # pyright: ignore[reportAttributeAccessIssue] # fixture module built at test time + monkeypatch.setitem(sys.modules, mcp_module.__name__, mcp_module) + features: Final = tuple( + replace(feature, module_path=mcp_module.__name__) if feature.name == "mcp_app" else feature + for feature in LAZY_FEATURES + if feature.name in ("mcp_app", "realtime") + ) + + async def mcp_proxy() -> Mapping[str, str]: + return {"route": "mcp_proxy"} + + app: Final = FastAPI() + attach_lazy_features(app, features) + app.add_api_route("/mcp/proxy", mcp_proxy, methods=["GET"]) + + with TestClient(app) as client: + for path in load_order: + client.get(path) + response: Final = client.get("/mcp/proxy") + + assert loaded_lazy_modules(app) == {feature.module_path for feature in features} + assert response.json() == {"route": "mcp_proxy"} + route_ids: Final = tuple(id(route) for route in app.routes) + assert len(route_ids) == len(frozenset(route_ids)) diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 26280ad4523..78adb46698e 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1,8 +1,12 @@ import asyncio +import ipaddress import json +import ssl +import threading import time -from collections.abc import Awaitable, Callable, Mapping -from datetime import datetime, timezone +from collections.abc import Awaitable, Callable, Iterator, Mapping +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from types import TracebackType from typing import Final @@ -11,17 +15,23 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID import litellm from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.litellm_core_utils.realtime_streaming import TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, RealTimeStreaming +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai.workload_identity import _workload_identity_auth from litellm.models.credentials import CredentialItem from litellm.realtime_api import main as realtime_main from litellm.realtime_api.main import _with_resolved_session_model from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.realtime import RealtimeQueryParams @pytest.fixture @@ -745,28 +755,39 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) -@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) -def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map, model: str) -> None: +_GA_DEFAULT_SESSIONS: Final = ( + pytest.param("gpt-realtime-2.1", "realtime", None, id="ga-only-model"), + pytest.param("gpt-realtime-2", "realtime", None, id="ga-only-alias"), + pytest.param("gpt-realtime-2-2026-05-06", "realtime", None, id="ga-only-snapshot"), + pytest.param("my-transcribe-deployment", "realtime", {"intent": "transcription"}, id="transcription-intent"), + pytest.param("my-translate-deployment", "translation", None, id="translation-mode"), +) + + +@pytest.mark.parametrize(("model", "realtime_mode", "query_params"), _GA_DEFAULT_SESSIONS) +def test_azure_realtime_protocol_defaults_to_ga_when_none_is_configured( + local_model_cost_map, model: str, realtime_mode: str, query_params: RealtimeQueryParams | None +) -> None: from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client - assert ( - azure_realtime_protocol_for_client( - None, - model=model, - realtime_mode="realtime", - query_params=None, - websocket=_BETA_CLIENT, - ) - == "GA" + protocol: Final = azure_realtime_protocol_for_client( + None, model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_BETA_CLIENT ) - with pytest.raises(ValueError, match="requires the Azure OpenAI v1 Realtime API"): - azure_realtime_protocol_for_client( - "beta", - model=model, - realtime_mode="realtime", - query_params=None, - websocket=_BETA_CLIENT, - ) + + assert protocol == "GA" + + +@pytest.mark.parametrize(("model", "realtime_mode", "query_params"), _GA_DEFAULT_SESSIONS) +def test_azure_realtime_protocol_configured_by_the_operator_wins( + local_model_cost_map, model: str, realtime_mode: str, query_params: RealtimeQueryParams | None +) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + protocol: Final = azure_realtime_protocol_for_client( + "beta", model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_GA_CLIENT + ) + + assert protocol == "beta" async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: @@ -1278,3 +1299,348 @@ async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_key model_params={"model": "openai/gpt-realtime", **federated_deployment}, ) assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer federated-bearer"} + + +def _realtime_http_route_call( + route: str, + logging_obj: Logging, + custom_llm_provider: str, + api_base: str, + client: AsyncHTTPHandler | None = None, +) -> Awaitable[httpx.Response]: + if route == "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + custom_llm_provider=custom_llm_provider, + api_key="sk-test", + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + if route == "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + custom_llm_provider=custom_llm_provider, + api_key="sk-test", + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + if route == "calls": + return realtime_main.arealtime_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + return realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime-translate", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + + +def _realtime_http_logging(route: str) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id=f"test-realtime-{route}", + function_id=f"test-realtime-{route}", + ) + + +_OPENAI_REALTIME_HTTP_ROUTE_PATHS: Final = { + "client_secrets": "/v1/realtime/client_secrets", + "translation_client_secrets": "/v1/realtime/translations/client_secrets", + "calls": "/v1/realtime/calls", + "translation_calls": "/v1/realtime/translations/calls", +} + + +def _validity(builder: x509.CertificateBuilder) -> x509.CertificateBuilder: + now: Final = datetime.now(timezone.utc) + return ( + builder.serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=5)) + .not_valid_after(now + timedelta(hours=1)) + ) + + +def _write_private_ca_and_loopback_certificate(directory: Path) -> tuple[Path, Path, Path]: + ca_key: Final = ec.generate_private_key(ec.SECP256R1()) + server_key: Final = ec.generate_private_key(ec.SECP256R1()) + ca_name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "litellm realtime test CA")]) + ca_certificate: Final = ( + _validity(x509.CertificateBuilder()) + .subject_name(ca_name) + .issuer_name(ca_name) + .public_key(ca_key.public_key()) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension( + x509.KeyUsage( + digital_signature=True, + content_commitment=False, + key_encipherment=False, + data_encipherment=False, + key_agreement=False, + key_cert_sign=True, + crl_sign=True, + encipher_only=False, + decipher_only=False, + ), + critical=True, + ) + .add_extension(x509.SubjectKeyIdentifier.from_public_key(ca_key.public_key()), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + server_certificate: Final = ( + _validity(x509.CertificateBuilder()) + .subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])) + .issuer_name(ca_name) + .public_key(server_key.public_key()) + .add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False) + .add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_key.public_key()), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + ca_path: Final = directory / "ca.pem" + certificate_path: Final = directory / "server.pem" + key_path: Final = directory / "server.key" + ca_path.write_bytes(ca_certificate.public_bytes(serialization.Encoding.PEM)) + certificate_path.write_bytes(server_certificate.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + server_key.private_bytes( + serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption() + ) + ) + return ca_path, certificate_path, key_path + + +class _EchoPathHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + body: Final = json.dumps({"value": "ek_test", "path": self.path}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +@pytest.fixture +def private_ca_upstream(tmp_path: Path) -> Iterator[tuple[str, Path]]: + ca_path, certificate_path, key_path = _write_private_ca_and_loopback_certificate(tmp_path) + tls: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls.load_cert_chain(certificate_path, key_path) + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _EchoPathHandler) + server.socket = tls.wrap_socket(server.socket, server_side=True) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"https://127.0.0.1:{server.server_address[1]}", ca_path + server.shutdown() + server.server_close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", tuple(_OPENAI_REALTIME_HTTP_ROUTE_PATHS)) +async def test_openai_realtime_http_routes_trust_the_ca_bundle_set_in_litellm_ssl_verify( + route: str, private_ca_upstream: tuple[str, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + api_base, ca_path = private_ca_upstream + monkeypatch.setattr(litellm, "ssl_verify", str(ca_path)) + + response: Final = await _realtime_http_route_call( + route, _realtime_http_logging(route), custom_llm_provider="openai", api_base=api_base + ) + + assert response.status_code == 200 + assert response.json() == {"value": "ek_test", "path": _OPENAI_REALTIME_HTTP_ROUTE_PATHS[route]} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", tuple(_OPENAI_REALTIME_HTTP_ROUTE_PATHS)) +async def test_openai_realtime_http_routes_surface_the_upstream_error_body_verbatim(route: str) -> None: + upstream_error: Final = { + "error": { + "message": "Invalid model", + "type": "invalid_request_error", + "param": "session.model", + "code": "invalid_value", + } + } + + with pytest.raises(BaseLLMException) as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler( + transport=httpx.MockTransport(lambda request: httpx.Response(400, json=upstream_error)) + ), + ) + + assert raised.value.status_code == 400 + assert json.loads(raised.value.message) == upstream_error + + +class _GatewayThatRecordsTheSession: + session: Mapping[str, object] | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.session = json.loads(request.content)["session"] + return httpx.Response(200, json={"value": "ek_test"}) + + +@pytest.mark.asyncio +async def test_client_secret_keeps_the_realtime_session_model_when_input_transcription_names_another() -> None: + gateway: Final = _GatewayThatRecordsTheSession() + logging_obj: Final = _realtime_http_logging("client_secrets") + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + session={ + "type": "realtime", + "model": "gpt-realtime", + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, + }, + custom_llm_provider="openai", + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=logging_obj, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-realtime" + assert gateway.session["audio"] == {"input": {"transcription": {"model": "gpt-realtime-whisper"}}} + assert logging_obj.model == "gpt-realtime" + + +class _UpstreamThatMustNotBeCalled: + called: bool = False + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.called = True + return httpx.Response(200, json={"value": "ek_test"}) + + +class _RecordingClientWebSocket: + def __init__(self) -> None: + self.scope: Final = {"headers": ()} + self.sent: tuple[str, ...] = () + self.close_code: int | None = None + + async def send_text(self, data: str) -> None: + self.sent = (*self.sent, data) + + async def close(self, code: int = 1000, reason: str = "") -> None: + self.close_code = code + + +def _block_every_request_with_a_guardrail(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + litellm, + "callbacks", + [ + CustomGuardrail( + guardrail_name="translation-blocker", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + ], + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) +async def test_translation_http_routes_refuse_a_guarded_request_before_any_upstream_call( + route: str, monkeypatch: pytest.MonkeyPatch +) -> None: + _block_every_request_with_a_guardrail(monkeypatch) + upstream: Final = _UpstreamThatMustNotBeCalled() + + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + ) + + assert raised.value.status_code == 400 + assert upstream.called is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) +async def test_translation_http_routes_refuse_a_provider_other_than_openai_and_azure_before_any_upstream_call( + route: str, +) -> None: + upstream: Final = _UpstreamThatMustNotBeCalled() + + with pytest.raises(litellm.BadRequestError, match="supports only the azure, openai providers, got xai") as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="xai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + ) + + assert raised.value.status_code == 400 + assert upstream.called is False + + +@pytest.mark.asyncio +async def test_translation_websocket_closes_1008_for_a_guarded_request_before_the_upstream_connect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + ) + + assert connect.url is None + assert websocket.close_code == 1008 + assert [json.loads(event)["error"]["message"] for event in websocket.sent] == [ + TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE + ] + + +@pytest.mark.asyncio +async def test_translation_websocket_closes_1008_for_a_provider_other_than_openai_and_azure() -> None: + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="xai/grok-voice", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + ) + + assert connect.url is None + assert websocket.close_code == 1008 + assert [json.loads(event)["error"]["message"] for event in websocket.sent] == [ + "Realtime translation supports only the azure, azure_ai, openai providers, got xai" + ] diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 7625bb03191..017c5a1364c 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -9878,6 +9878,127 @@ def test_responses_ws_logging_object_carries_the_completed_responses_id_tier_and assert ResponsesWebSocketTokenUsageProcessor.create_logging_object([{"type": "error", "error": {}}]).id is None +@pytest.mark.parametrize("provider", ("openai", "azure")) +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str): + model_name: Final = f"azure/{model}" if provider == "azure" else model + rates: Final = litellm.model_cost[model_name] + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.created", "session": {"model": model}}, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 300, + "total_tokens": 1300, + "input_token_details": { + "text_tokens": 400, + "audio_tokens": 400, + "image_tokens": 200, + "cached_tokens": 300, + "cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50}, + }, + "output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100}, + } + }, + }, + ] + combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events) + actual: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=combined, + custom_llm_provider=provider, + litellm_model_name=model_name, + ) + expected: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + + 200 * rates["output_cost_per_token"] + + 100 * rates["output_cost_per_audio_token"] + ) + + assert actual == pytest.approx(expected) + + +def test_realtime_translation_duration_cost(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) + + +def test_realtime_translation_bills_session_audio_once_on_openai_and_azure(_local_model_cost_map) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + one_minute_session: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}} + ] + openai_cost: Final = handle_realtime_translation_cost_calculation( + results=one_minute_session, custom_llm_provider="openai", litellm_model_name="gpt-realtime-translate" + ) + azure_cost: Final = handle_realtime_translation_cost_calculation( + results=one_minute_session, custom_llm_provider="azure", litellm_model_name="gpt-realtime-translate" + ) + + assert openai_cost == pytest.approx(60 * litellm.model_cost["gpt-realtime-translate"]["output_cost_per_second"]) + assert openai_cost == pytest.approx(azure_cost) + + +@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( + _local_model_cost_map, malformed_duration: float | int | bool +) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "translation-deployment-priced-both-directions" + litellm.register_model( + model_cost={ + model: { + "litellm_provider": "openai", + "mode": "realtime", + "input_cost_per_second": 0.001, + "output_cost_per_second": 0.002, + } + } + ) + input_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5}, + } + ] + output_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration}, + } + ] + + input_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + output_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + + assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"]) + assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) + + @pytest.mark.parametrize("profile_prefix", ("us", "global")) @pytest.mark.parametrize( "model_name", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 80368fe1e73..b545a0f090f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11084,46 +11084,6 @@ export interface paths { patch?: never; trace?: never; }; - "/openai/v1/realtime/translations/calls": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Proxy Realtime Calls - * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets - */ - post: operations["proxy_realtime_calls_openai_v1_realtime_translations_calls_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; - "/openai/v1/realtime/translations/client_secrets": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - get?: never; - put?: never; - /** - * Create Realtime Client Secret - * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session - */ - post: operations["create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post"]; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/openai/v1/responses": { parameters: { query?: never; @@ -48064,6 +48024,54 @@ export interface components { */ success_definition: string; }; + /** TranscriptionDetectedLanguage */ + TranscriptionDetectedLanguage: { + /** Code */ + code: string; + }; + /** TranscriptionResponse */ + TranscriptionResponse: { + /** Languages */ + languages?: components["schemas"]["TranscriptionDetectedLanguage"][] | null; + /** Text */ + text?: string | null; + /** Usage */ + usage?: components["schemas"]["TranscriptionUsageDurationObject"] | components["schemas"]["TranscriptionUsageTokensObject"] | null; + } & { + [key: string]: unknown; + }; + /** TranscriptionUsageDurationObject */ + TranscriptionUsageDurationObject: { + /** Seconds */ + seconds: number; + /** + * Type + * @constant + */ + type: "duration"; + }; + /** TranscriptionUsageInputTokenDetailsObject */ + TranscriptionUsageInputTokenDetailsObject: { + /** Audio Tokens */ + audio_tokens: number; + /** Text Tokens */ + text_tokens: number; + }; + /** TranscriptionUsageTokensObject */ + TranscriptionUsageTokensObject: { + input_token_details?: components["schemas"]["TranscriptionUsageInputTokenDetailsObject"] | null; + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + /** Total Tokens */ + total_tokens: number; + /** + * Type + * @constant + */ + type: "tokens"; + }; /** TransformRequestBody */ TransformRequestBody: { call_type: components["schemas"]["CallTypes"]; @@ -52777,7 +52785,8 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["TranscriptionResponse"]; + "text/event-stream": string; }; }; /** @description Validation Error */ @@ -66920,156 +66929,6 @@ export interface operations { }; }; }; - proxy_realtime_calls_openai_v1_realtime_translations_calls_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/sdp": string; - }; - }; - responses: { - /** @description WebRTC SDP answer */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": unknown; - "application/sdp": string; - }; - }; - }; - }; - create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - /** - * @example { - * "model": "gpt-realtime-translate", - * "session": { - * "model": "gpt-realtime-translate", - * "type": "translation" - * } - * } - */ - "application/json": { - expires_after?: { - /** - * Anchor - * @default created_at - */ - anchor?: string | null; - /** Seconds */ - seconds?: number | null; - } | null; - /** Model */ - model?: string | null; - session?: ({ - audio?: ({ - input?: ({ - /** Format */ - format?: string | { - [key: string]: unknown; - } | null; - /** Noise Reduction */ - noise_reduction?: { - [key: string]: unknown; - } | null; - transcription?: ({ - /** Delay */ - delay?: ("minimal" | "low" | "medium" | "high" | "xhigh") | null; - /** Keywords */ - keywords?: string[] | null; - /** Language */ - language?: string | null; - /** Languages */ - languages?: string[] | null; - /** Model */ - model?: string | null; - /** Prompt */ - prompt?: string | null; - } & { - [key: string]: unknown; - }) | null; - /** Turn Detection */ - turn_detection?: { - [key: string]: unknown; - } | null; - } & { - [key: string]: unknown; - }) | null; - output?: ({ - /** Format */ - format?: string | { - [key: string]: unknown; - } | null; - /** Language */ - language?: string | null; - /** Voice */ - voice?: string | { - [key: string]: unknown; - } | null; - } & { - [key: string]: unknown; - }) | null; - } & { - [key: string]: unknown; - }) | null; - /** Include */ - include?: string[] | null; - /** Instructions */ - instructions?: string | null; - /** Max Output Tokens */ - max_output_tokens?: number | string | null; - /** Model */ - model?: string | null; - /** Output Modalities */ - output_modalities?: string[] | null; - /** Prompt */ - prompt?: { - [key: string]: unknown; - } | null; - /** Tool Choice */ - tool_choice?: unknown | null; - /** Tools */ - tools?: { - [key: string]: unknown; - }[] | null; - /** Tracing */ - tracing?: unknown | null; - /** Truncation */ - truncation?: unknown | null; - /** Type */ - type?: ("realtime" | "transcription" | "translation") | null; - } & { - [key: string]: unknown; - }) | null; - }; - }; - }; - responses: { - /** @description Successful Response */ - 200: { - headers: { - [name: string]: unknown; - }; - content: { - "application/json": components["schemas"]["RealtimeClientSecretResponse"]; - }; - }; - }; - }; responses_api_openai_v1_responses_post: { parameters: { query?: never; @@ -70457,13 +70316,12 @@ export interface operations { }; responses: { /** @description WebRTC SDP answer */ - 200: { + 201: { headers: { [name: string]: unknown; }; content: { - "application/json": unknown; - "application/sdp": string; + "text/plain": string; }; }; }; @@ -77823,7 +77681,8 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["TranscriptionResponse"]; + "text/event-stream": string; }; }; /** @description Validation Error */ @@ -81527,13 +81386,12 @@ export interface operations { }; responses: { /** @description WebRTC SDP answer */ - 200: { + 201: { headers: { [name: string]: unknown; }; content: { - "application/json": unknown; - "application/sdp": string; + "text/plain": string; }; }; }; From c57d273aa6789e17d9c6a6cf42c2e74c6ca3f054 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:42:38 -0700 Subject: [PATCH 40/45] fix(realtime): restore main behavior for guardrails, Azure transcription, and realtime clients Streamed transcription runs unstreamed when a deployment post_call guardrail applies, so the guardrail still sees the transcript. Azure transcription maps v1, latest and preview to the default dated version on the deployments route for every model. The SDK client secret keeps session.model ahead of model, realtime WebSockets connect to api_base again, realtime metadata keeps only guardrails, and managed agents need an explicit transcription model. Reverts the two unrelated streamed Responses error codes and the duplicate cached_tokens_details field, and the gpt-transcribe format check only reads the request provider's cost map entry. --- litellm/integrations/custom_guardrail.py | 13 +- litellm/llms/azure/audio_transcriptions.py | 12 +- litellm/llms/openai/realtime/handler.py | 120 +------- litellm/main.py | 7 +- ...odel_prices_and_context_window_backup.json | 1 - .../auth/managed_authorization.py | 8 +- litellm/proxy/auth/user_api_key_auth.py | 1 - litellm/realtime_api/main.py | 30 +- litellm/responses/streaming_iterator.py | 2 - litellm/router.py | 6 +- litellm/types/utils.py | 2 - litellm/utils.py | 17 +- model_prices_and_context_window.json | 1 - .../llms/azure/test_audio_transcriptions.py | 61 ++-- .../realtime/test_openai_realtime_handler.py | 278 ++++++++++-------- .../realtime/test_transcription_sessions.py | 8 +- .../auth/test_managed_authorization.py | 15 +- .../test_user_api_key_auth_request_flow.py | 41 +++ .../proxy/common_utils/test_realtime_cache.py | 14 - .../proxy/proxy_server/test_routes_audio.py | 85 +++++- .../test_realtime_webrtc_endpoints.py | 102 ++++++- tests/unit/realtime_api/test_main.py | 104 ++++++- .../test_streaming_iterator_error_events.py | 2 - tests/unit/test_main.py | 36 +++ 24 files changed, 601 insertions(+), 365 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 1c12ac395c7..9d7b699333b 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -974,6 +974,12 @@ class CustomGuardrail(CustomLogger): enrich_http_exception_with_guardrail_context(result, self) raise result + def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool: + return ( + isinstance(request_data.get("guardrails"), list) + and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True + ) + async def async_post_call_success_deployment_hook( self, request_data: dict, @@ -983,12 +989,7 @@ class CustomGuardrail(CustomLogger): """ Allow modifying / reviewing the response just after it's received from the deployment. """ - # should run guardrail - litellm_guardrails: Final = request_data.get("guardrails") - if litellm_guardrails is None or not isinstance(litellm_guardrails, list): - return None - - if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True: + if not self.runs_post_call_deployment_hook(request_data): return None target: Final = self._deployment_hook_target() diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 90c240a611c..8f5a26d41a4 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -12,7 +12,6 @@ from pydantic import BaseModel import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name -from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.audio_transcription.transformation import ( sdk_compatible_transcription_request_data, sdk_compatible_transcription_response, @@ -56,17 +55,8 @@ class AzureAudioTranscription(AzureChatCompletion): ): data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) - model_info: Final = as_str_mapping(litellm.model_cost.get(f"azure/{model}")) - provider_specific_entry: Final = ( - as_str_mapping(model_info.get("provider_specific_entry")) if model_info is not None else None - ) - requires_deployment_api: Final = ( - provider_specific_entry is not None and provider_specific_entry.get("transcription_deployment_api") == 1 - ) resolved_api_version: Final = ( - litellm.AZURE_DEFAULT_API_VERSION - if requires_deployment_api and api_version in ("v1", "latest", "preview") - else api_version + litellm.AZURE_DEFAULT_API_VERSION if api_version in ("v1", "latest", "preview") else api_version ) if atranscription is True: diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index b640d4d9c14..8951dd70cef 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -5,19 +5,8 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ import ssl -from collections.abc import Mapping -from contextlib import AbstractAsyncContextManager -from types import MappingProxyType, TracebackType from typing import Any, Final, cast -from openai import AsyncOpenAI, omit -from openai.resources.realtime.realtime import ( - AsyncRealtimeConnection, - AsyncRealtimeConnectionManager, -) -from openai.types.websocket_connection_options import WebsocketConnectionOptions as WebSocketConnectionOptions -from typing_extensions import NotRequired, ReadOnly - from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams @@ -35,54 +24,6 @@ from ..common_utils import is_openai_backed_api_base from ..openai import OpenAIChatCompletion -class _RealtimeConnectionOptions(WebSocketConnectionOptions): - open_timeout: NotRequired[ReadOnly[float]] - ssl: NotRequired[ReadOnly[bool | str | ssl.SSLContext | None]] - - -class OpenAIRealtimeConnectionAdapter: - def __init__(self, connection: AsyncRealtimeConnection) -> None: - self._connection = connection - - async def send(self, message: str) -> None: - await self._connection.send_raw(message) - - async def recv(self, decode: bool = True) -> str | bytes: - message: Final = await self._connection.recv_bytes() - if decode: - return message.decode("utf-8") - return message - - async def close(self) -> None: - await self._connection.close() - - -class OpenAIRealtimeSDKConnectionManager: - def __init__( - self, - manager: AsyncRealtimeConnectionManager, - owned_client: AsyncOpenAI | None = None, - ) -> None: - self._manager = manager - self._owned_client = owned_client - - async def __aenter__(self) -> OpenAIRealtimeConnectionAdapter: - connection: Final = await self._manager.__aenter__() - return OpenAIRealtimeConnectionAdapter(connection) - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc: BaseException | None, - traceback: TracebackType | None, - ) -> None: - try: - await self._manager.__aexit__(exc_type, exc, traceback) - finally: - if self._owned_client is not None: - await self._owned_client.close() - - class OpenAIRealtime(OpenAIChatCompletion): """ Base handler for OpenAI-compatible realtime WebSocket connections. @@ -188,43 +129,6 @@ class OpenAIRealtime(OpenAIChatCompletion): """Return a per-session usage meter, or None when billing needs no metering.""" return None - def _create_connection_manager( - self, - api_base: str, - api_key: str, - model: str, - query_params: RealtimeQueryParams, - headers: Mapping[str, str], - timeout: float | None, - realtime_mode: str, - ssl_config: bool | str | ssl.SSLContext | None, - client: object | None, - url: str, - ) -> AbstractAsyncContextManager[object]: - import websockets - - timeout_options: Final[_RealtimeConnectionOptions] = {"open_timeout": timeout} if timeout is not None else {} - ssl_options: Final[_RealtimeConnectionOptions] = {"ssl": ssl_config} if url.startswith("wss://") else {} - connection_options: Final[_RealtimeConnectionOptions] = { - "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - **ssl_options, - **timeout_options, - } - if realtime_mode == "translation" or not isinstance(client, AsyncOpenAI): - return websockets.connect(url, additional_headers=headers, **connection_options) - openai_client: Final = client - model_query: Final = query_params.get("model") - extra_query: Final = MappingProxyType({key: value for key, value in query_params.items() if key != "model"}) - sdk_model: Final = omit if query_params.get("intent") == "transcription" else model_query or model - sdk_connection_manager: Final = openai_client.realtime.connect( - model=sdk_model, - extra_query=extra_query, - extra_headers=headers, - websocket_connection_options=connection_options, - max_retries=0, - ) - return OpenAIRealtimeSDKConnectionManager(sdk_connection_manager) - async def async_realtime( self, model: str, @@ -278,25 +182,15 @@ class OpenAIRealtime(OpenAIChatCompletion): "complete_input_dict": {"query_params": query_params}, }, ) - connection_manager: Final = self._create_connection_manager( - api_base=api_base, - api_key=api_key, - model=model, - query_params=query_params, - headers=headers, - timeout=timeout, - realtime_mode=realtime_mode, - ssl_config=ssl_config, - client=client, - url=url, - ) - - async with connection_manager as backend_ws: + async with websockets.connect( + url, + additional_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=ssl_config, + ) as backend_ws: realtime_streaming: Final = RealTimeStreaming( websocket, - cast( # cast-ok: both SDK and websockets adapters implement the streaming connection interface - ClientConnection, backend_ws - ), + cast(ClientConnection, backend_ws), logging_obj, model=model, user_api_key_dict=user_api_key_dict, diff --git a/litellm/main.py b/litellm/main.py index b38a33fbd4f..b8e39c2b18f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8259,7 +8259,12 @@ def _validate_gpt_transcription_request( drop_params: object, ) -> str | None: model_cost_key: Final = next( - (key for key in (f"{custom_llm_provider}/{model}", model) if key in litellm.model_cost), None + ( + key + for key in (f"{custom_llm_provider}/{model}", model) + if (as_str_mapping(litellm.model_cost.get(key)) or {}).get("litellm_provider") == custom_llm_provider + ), + None, ) model_info: Final = ( get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_cost_key is not None else None diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7c5dfad023a..1e649b43110 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -71188,7 +71188,6 @@ "supports_audio_input": true, "provider_specific_entry": { "realtime_ga_only": 1, - "transcription_deployment_api": 1, "transcription_json_or_text_only": 1, "transcription_usage_duration": 1 }, diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 3740d157249..427ed071dc4 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -115,7 +115,6 @@ def managed_inference_request( cli_model: str | None, path_model: object = None, query_model: object = None, - intent: str | None = None, ) -> dict[str, object]: from litellm.proxy.auth.route_checks import RouteChecks @@ -124,11 +123,8 @@ def managed_inference_request( if route in _MANAGED_REALTIME_ROUTES: from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model - requested_model: Final = query_model if query_model is not None else body.get("model") - model: Final = ( - resolve_realtime_route_model(None, intent, route.endswith("/realtime/translations")) - if requested_model is None - else requested_model + model: Final = resolve_realtime_route_model( + query_model if isinstance(query_model, str) else None, None, route.endswith("/realtime/translations") ) if not isinstance(model, str) or not model: raise_identity_failure( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b51da241318..07c4626ac23 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -4072,7 +4072,6 @@ async def _authorize_authenticated_request( user_model, request.path_params.get("model") or request.path_params.get("model_name"), request.query_params.get("model"), - intent=request.query_params.get("intent"), ) if user_api_key_auth_obj.managed_agent_policy is not None else request_data diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 20c24e03a76..f11714218a8 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -85,7 +85,7 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> return MappingProxyType({**credential_values, **model_params}) -def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: +def with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: if session.get("type") == "transcription": audio: Final = as_str_mapping(session.get("audio")) or {} audio_input: Final = as_str_mapping(audio.get("input")) or {} @@ -108,11 +108,17 @@ def _with_resolved_session_model(session: dict[str, object], model_name: str) -> return {**session, "model": model_name} +_POLICY_PIPELINE_METADATA_KEYS: Final = ("_guardrail_pipelines", "_pipeline_managed_guardrails") + + def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]: request_metadata: Final = _METADATA_ADAPTER.validate_python(kwargs.get("metadata") or {}) internal_metadata: Final = _METADATA_ADAPTER.validate_python(kwargs.get("litellm_metadata") or {}) + policy_pipeline_state: Final = { + key: request_metadata[key] for key in _POLICY_PIPELINE_METADATA_KEYS if key in request_metadata + } guardrails: Final = request_metadata.get("guardrails") or kwargs.get("guardrails") - return {**request_metadata, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})} + return {**policy_pipeline_state, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})} _HTTP_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) @@ -265,16 +271,8 @@ async def acreate_realtime_client_secret( and req.session.audio.input.transcription is not None else None ) - provider_qualified_model: Final = ( - req.model - if req.model is not None - and "/" in req.model - and req.model.split("/", 1)[0] in LlmProviders._member_map_.values() - else None - ) requested_model_name: Final = ( - provider_qualified_model - or transcription_model + transcription_model or (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview" @@ -309,7 +307,7 @@ async def acreate_realtime_client_secret( ) request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) if isinstance(request_data.get("session"), dict): - request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + request_data["session"] = with_resolved_session_model(request_data["session"], model_name) elif req.model is not None: request_data["session"] = { "type": "realtime", @@ -390,7 +388,7 @@ async def acreate_realtime_translation_client_secret( exclude_none=True, exclude={"model"}, ) - request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + request_data["session"] = with_resolved_session_model(request_data["session"], model_name) request_data["session"].pop("type", None) return await base_llm_http_handler.async_realtime_translation_client_secret_handler( api_base=resolved_api_base, @@ -520,7 +518,7 @@ async def arealtime_calls( litellm_params=litellm_params, ) if session is not None: - session = _with_resolved_session_model(session, model_name) + session = with_resolved_session_model(session, model_name) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -571,7 +569,7 @@ async def arealtime_translation_calls( dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) - session_config: Final = _with_resolved_session_model( + session_config: Final = with_resolved_session_model( { **(session or {}), "type": "translation", @@ -880,7 +878,7 @@ async def _arealtime( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - client=client, + client=None, timeout=timeout, query_params=query_params, realtime_mode=realtime_mode, diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8b4863d7b22..0d59c93786c 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -227,8 +227,6 @@ _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( "insufficient_quota": 429, "vector_store_timeout": 504, "invalid_prompt": 400, - "data_residency_mismatch": 400, - "bio_policy": 400, "invalid_image": 400, "invalid_image_format": 400, "invalid_base64_image": 400, diff --git a/litellm/router.py b/litellm/router.py index 39c0bcda338..1d9ed822bc8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -709,9 +709,11 @@ def _with_router_resolved_session_model(session: object, model_name: str) -> Map typed_session: Final = _SESSION_ADAPTER.validate_python(session) except ValidationError: return _NO_SESSION_KWARGS - if "model" not in typed_session: + if "model" not in typed_session and typed_session.get("type") != "transcription": return _NO_SESSION_KWARGS - return MappingProxyType({"session": {**typed_session, "model": model_name}}) + from litellm.realtime_api.main import with_resolved_session_model + + return MappingProxyType({"session": with_resolved_session_model(dict(typed_session), model_name)}) # Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 80f31f382b7..9bc4fad6bf0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1832,8 +1832,6 @@ class PromptTokensDetailsWrapper( image_tokens: int | None = None """Image tokens sent to the model.""" - cached_tokens_details: CachedTokensDetails | None = None - video_tokens: int | None = None """Video tokens sent to the model.""" diff --git a/litellm/utils.py b/litellm/utils.py index d9aeaa53a6b..8b0d437775b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1658,7 +1658,22 @@ async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) if result is not None: modified_kwargs = result - return modified_kwargs + return _unstreamed_when_post_call_guardrails_apply(modified_kwargs, typed_call_type) + + +def _unstreamed_when_post_call_guardrails_apply( + kwargs: dict[str, object], call_type: CallTypes | None +) -> dict[str, object]: + if call_type is not CallTypes.atranscription or kwargs.get("stream") is not True: + return kwargs + unstreamed: Final = {**kwargs, "stream": False} + custom_guardrail_class: Final = _get_cached_custom_guardrail() + if not any( + isinstance(callback, custom_guardrail_class) and callback.runs_post_call_deployment_hook(unstreamed) + for callback in litellm.callbacks + ): + return kwargs + return unstreamed async def async_post_call_success_deployment_hook( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7c5dfad023a..1e649b43110 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -71188,7 +71188,6 @@ "supports_audio_input": true, "provider_specific_entry": { "realtime_ga_only": 1, - "transcription_deployment_api": 1, "transcription_json_or_text_only": 1, "transcription_usage_duration": 1 }, diff --git a/tests/unit/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py index f8a294c7953..675b7f8c1f1 100644 --- a/tests/unit/llms/azure/test_audio_transcriptions.py +++ b/tests/unit/llms/azure/test_audio_transcriptions.py @@ -1,18 +1,15 @@ -import io import json from pathlib import Path from typing import Final -from unittest.mock import MagicMock import httpx import pytest +import respx from openai import AsyncAzureOpenAI, AzureOpenAI import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration -from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription -from litellm.types.utils import TranscriptionResponse AUDIO_FILE: Final = Path(__file__).parents[3] / "gettysburg.wav" WHISPER_COST_PER_SECOND: Final = 0.0001 @@ -166,41 +163,37 @@ async def test_azure_transcription_parses_response_formats( assert response.text == expected_text -@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) +@pytest.mark.parametrize("model", ["whisper-1", "gpt-transcribe", "custom-transcribe-deployment"]) @pytest.mark.parametrize( - ("model", "expected_path"), + ("api_version", "expected_api_version"), [ - ("whisper-1", "/openai/v1/audio/transcriptions"), - ("gpt-transcribe", "/openai/deployments/gpt-transcribe/audio/transcriptions"), - ("custom-transcribe-deployment", "/openai/v1/audio/transcriptions"), + ("v1", litellm.AZURE_DEFAULT_API_VERSION), + ("latest", litellm.AZURE_DEFAULT_API_VERSION), + ("preview", litellm.AZURE_DEFAULT_API_VERSION), + ("2025-03-01-preview", "2025-03-01-preview"), ], ) -def test_azure_transcription_alias_uses_model_route( - monkeypatch: pytest.MonkeyPatch, model: str, expected_path: str, api_version: str +async def test_azure_transcription_uses_the_deployments_route( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + api_version: str, + expected_api_version: str, ) -> None: - def send_response(request: httpx.Request) -> httpx.Response: - assert request.url.path == expected_path - assert request.url.params.get("api-version") == ( - None if expected_path.startswith("/openai/v1/") else litellm.AZURE_DEFAULT_API_VERSION - ) - return httpx.Response(200, json={"text": "hello"}) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) - audio_file: Final = io.BytesIO(b"audio") - audio_file.name = "sample.wav" - with httpx.Client(transport=httpx.MockTransport(send_response)) as http_client: - monkeypatch.setattr(litellm, "client_session", http_client) - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - response: Final = AzureAudioTranscription().audio_transcriptions( - model=model, - audio_file=audio_file, - optional_params={"response_format": "json"}, - logging_obj=MagicMock(), - model_response=TranscriptionResponse(), - timeout=10, - max_retries=0, - api_key="test-key", - api_base="https://example.openai.azure.com", - api_version=api_version, - ) + response: Final = await litellm.atranscription( + model=f"azure/{model}", + file=_audio_file(), + api_key="test-key", + api_base="https://transcribe-route.openai.azure.com", + api_version=api_version, + ) + sent_url: Final = route.calls.last.request.url + assert sent_url.path == f"/openai/deployments/{model}/audio/transcriptions" + assert sent_url.params.get("api-version") == expected_api_version assert response.text == "hello" diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index 4e456e364d2..8ed65ef925c 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -5,39 +5,12 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest - -from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from pydantic import TypeAdapter from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from litellm.types.realtime import RealtimeQueryParams - -from openai import AsyncOpenAI, omit -from pydantic import TypeAdapter - - -class DummySDKConnectionManager: - def __init__(self, connection): - self.connection = connection - - async def __aenter__(self): - return self.connection - - async def __aexit__(self, exc_type, exc, tb): - return None - - -def make_realtime_sdk_client(): - connection = MagicMock() - connection.send_raw = AsyncMock() - connection.recv_bytes = AsyncMock() - connection.close = AsyncMock() - client = MagicMock(spec=AsyncOpenAI) - client.realtime.connect = MagicMock(return_value=DummySDKConnectionManager(connection)) - return client - - @pytest.mark.parametrize("api_base", ["https://api.openai.com/v1", "https://api.openai.com"]) def test_openai_realtime_handler_url_construction(api_base): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -130,10 +103,27 @@ async def test_async_realtime_success(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - sdk_client = make_realtime_sdk_client() - with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming: + mock_backend_ws = AsyncMock() + + class DummyAsyncContextManager: + def __init__(self, value): + self.value = value + + async def __aenter__(self): + return self.value + + async def __aexit__(self, exc_type, exc, tb): + return None + + shared_context = get_shared_realtime_ssl_context() + with ( + patch( + "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) + ) as mock_ws_connect, + patch( + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming, + ): mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -145,7 +135,6 @@ async def test_async_realtime_success(): api_base=api_base, api_key=api_key, query_params=query_params, - client=sdk_client, ) mock_realtime_streaming.assert_called_once() @@ -169,10 +158,28 @@ async def test_async_realtime_url_contains_model(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - sdk_client = make_realtime_sdk_client() - with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming: + mock_backend_ws = AsyncMock() + + class DummyAsyncContextManager: + def __init__(self, value): + self.value = value + + async def __aenter__(self): + return self.value + + async def __aexit__(self, exc_type, exc, tb): + return None + + shared_context = get_shared_realtime_ssl_context() + with ( + patch( + "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) + ) as mock_ws_connect, + patch( + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming, + ): + mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -184,48 +191,30 @@ async def test_async_realtime_url_contains_model(): api_base=api_base, api_key=api_key, query_params=query_params, - client=sdk_client, ) - sdk_client.realtime.connect.assert_called_once() - called_kwargs = sdk_client.realtime.connect.call_args.kwargs - assert called_kwargs["model"] == model - additional_headers = called_kwargs["extra_headers"] + # Verify websockets.connect was called with the correct URL + mock_ws_connect.assert_called_once() + called_url = mock_ws_connect.call_args[0][0] + + # Verify the URL contains the model parameter + assert called_url.startswith("wss://api.openai.com/v1/realtime?") + assert f"model={model}" in called_url + + # Verify proper headers were set (GA default: no OpenAI-Beta unless client sent it) + called_kwargs = mock_ws_connect.call_args[1] + assert "additional_headers" in called_kwargs + additional_headers = called_kwargs["additional_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert "OpenAI-Beta" not in additional_headers - assert called_kwargs["max_retries"] == 0 + # Verify SSL is configured (should be an SSLContext or True, not None or False) + assert called_kwargs["ssl"] is not None + assert called_kwargs["ssl"] is not False mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() -@pytest.mark.asyncio -async def test_async_realtime_transcription_omits_sdk_model_query(): - from litellm.llms.openai.realtime.handler import OpenAIRealtime - - handler = OpenAIRealtime() - websocket = AsyncMock() - logging_obj = MagicMock() - sdk_client = make_realtime_sdk_client() - with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming: - mock_realtime_streaming.return_value.bidirectional_forward = AsyncMock() - - await handler.async_realtime( - model="gpt-live-transcribe", - websocket=websocket, - logging_obj=logging_obj, - api_key="test-key", - query_params={"intent": "transcription"}, - client=sdk_client, - ) - - called_kwargs = sdk_client.realtime.connect.call_args.kwargs - assert called_kwargs["model"] is omit - assert called_kwargs["extra_query"] == {"intent": "transcription"} - - @pytest.mark.asyncio async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it(): """Upstream WS gets OpenAI-Beta: realtime=v1 only when the client WebSocket included it.""" @@ -245,10 +234,26 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() ] } dummy_logging_obj = MagicMock() - sdk_client = make_realtime_sdk_client() - with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming: + mock_backend_ws = AsyncMock() + + class DummyAsyncContextManager: + def __init__(self, value): + self.value = value + + async def __aenter__(self): + return self.value + + async def __aexit__(self, exc_type, exc, tb): + return None + + with ( + patch( + "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) + ) as mock_ws_connect, + patch( + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming, + ): mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -260,12 +265,11 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() api_base=api_base, api_key=api_key, query_params=query_params, - client=sdk_client, ) - sdk_client.realtime.connect.assert_called_once() - called_kwargs = sdk_client.realtime.connect.call_args.kwargs - additional_headers = called_kwargs["extra_headers"] + mock_ws_connect.assert_called_once() + called_kwargs = mock_ws_connect.call_args[1] + additional_headers = called_kwargs["additional_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert additional_headers["OpenAI-Beta"] == "realtime=v1" @@ -289,10 +293,28 @@ async def test_async_realtime_uses_max_size_parameter(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - sdk_client = make_realtime_sdk_client() - with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming: + mock_backend_ws = AsyncMock() + + class DummyAsyncContextManager: + def __init__(self, value): + self.value = value + + async def __aenter__(self): + return self.value + + async def __aexit__(self, exc_type, exc, tb): + return None + + shared_context = get_shared_realtime_ssl_context() + with ( + patch( + "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) + ) as mock_ws_connect, + patch( + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming, + ): + mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -304,14 +326,20 @@ async def test_async_realtime_uses_max_size_parameter(): api_base=api_base, api_key=api_key, query_params=query_params, - client=sdk_client, ) - sdk_client.realtime.connect.assert_called_once() - called_kwargs = sdk_client.realtime.connect.call_args.kwargs - connection_options = called_kwargs["websocket_connection_options"] - assert connection_options["max_size"] is REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES - assert connection_options["ssl"] is not None + # Verify websockets.connect was called with the max_size parameter + mock_ws_connect.assert_called_once() + called_kwargs = mock_ws_connect.call_args[1] + + # Verify max_size is set (default None for unlimited, matching OpenAI's SDK) + assert "max_size" in called_kwargs + assert called_kwargs["max_size"] is None + # Verify SSL is configured (should be an SSLContext or True, not None or False) + assert called_kwargs["ssl"] is not None + assert called_kwargs["ssl"] is not False + # Default should be None (unlimited) to match OpenAI's official agents SDK + # https://github.com/openai/openai-agents-python/blob/cf1b933660e44fd37b4350c41febab8221801409/src/agents/realtime/openai_realtime.py#L235 mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() @@ -336,10 +364,27 @@ async def test_async_realtime_ws_url_has_no_ssl(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - sdk_client = make_realtime_sdk_client() - with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming: + mock_backend_ws = AsyncMock() + + class DummyAsyncContextManager: + def __init__(self, value): + self.value = value + + async def __aenter__(self): + return self.value + + async def __aexit__(self, exc_type, exc, tb): + return None + + with ( + patch( + "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) + ) as mock_ws_connect, + patch( + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming, + ): + mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -351,13 +396,19 @@ async def test_async_realtime_ws_url_has_no_ssl(): api_base=api_base, api_key=api_key, query_params=query_params, - client=sdk_client, ) - sdk_client.realtime.connect.assert_called_once() - called_kwargs = sdk_client.realtime.connect.call_args.kwargs - assert called_kwargs["model"] == model - assert "ssl" not in called_kwargs["websocket_connection_options"] + # Verify websockets.connect was called + mock_ws_connect.assert_called_once() + called_url = mock_ws_connect.call_args[0][0] + called_kwargs = mock_ws_connect.call_args[1] + + # Verify URL was converted from http:// to ws:// + assert called_url.startswith("ws://localhost:8113/v1/realtime?") + assert f"model={model}" in called_url + + # Verify ssl is None for ws:// URLs (the fix for issue #19222) + assert called_kwargs["ssl"] is None @pytest.mark.asyncio @@ -480,6 +531,18 @@ async def test_realtime_query_params_preserve_missing_model(monkeypatch): called_kwargs = mock_async_realtime.call_args.kwargs assert called_kwargs["query_params"] == {"intent": "transcription"} + +class _BackendConnection: + def __init__(self, connection): + self.connection = connection + + async def __aenter__(self): + return self.connection + + async def __aexit__(self, exc_type, exc, tb): + return None + + def test_translation_url_uses_dedicated_path(): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -531,7 +594,7 @@ async def test_translation_websocket_uses_direct_transport(beta_client: bool) -> == expected_url ) - with patch("websockets.connect", return_value=DummySDKConnectionManager(backend)) as connect: + with patch("websockets.connect", return_value=_BackendConnection(backend)) as connect: await handler.async_realtime( model="gpt-realtime-translate", websocket=websocket, @@ -547,22 +610,3 @@ async def test_translation_websocket_uses_direct_transport(beta_client: bool) -> assert "OpenAI-Beta" not in connect.call_args.kwargs["additional_headers"] sent: Final = TypeAdapter(Mapping[str, object]).validate_json(backend.send.call_args_list[0].args[0]) assert sent["session"] == {"audio": {"output": {"language": "fr"}}} - - -@pytest.mark.parametrize("sdk_client", [True, False]) -def test_connection_manager_preserves_transport_settings(sdk_client: bool): - import ssl - from litellm.llms.openai.realtime.handler import OpenAIRealtime - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - - client = make_realtime_sdk_client() if sdk_client else MagicMock(spec=AsyncHTTPHandler) - ssl_config = ssl.create_default_context() - with patch("websockets.connect") as connect: - OpenAIRealtime()._create_connection_manager( - api_base="https://example.com", api_key="test", model="gpt-realtime-2.1", - query_params={"model": "gpt-realtime-2.1"}, headers={}, timeout=7.0, - realtime_mode="realtime", ssl_config=ssl_config, client=client, url="wss://example.com/v1/realtime", - ) - options = client.realtime.connect.call_args.kwargs["websocket_connection_options"] if sdk_client else connect.call_args.kwargs - assert options["ssl"] is ssl_config - assert options["open_timeout"] == 7.0 diff --git a/tests/unit/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py index 514965cd581..e2177268f6c 100644 --- a/tests/unit/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/unit/llms/openai/realtime/test_transcription_sessions.py @@ -11,7 +11,6 @@ import httpx import pytest import litellm - from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -152,16 +151,13 @@ async def test_client_secret_handler_still_targets_client_secrets_url(): @pytest.mark.asyncio -async def test_azure_client_secret_prefers_provider_qualified_routing_model(): - import litellm - +async def test_azure_client_secret_routes_on_a_provider_qualified_session_model(): mock_response = MagicMock(spec=httpx.Response) mock_client = MagicMock(spec=AsyncHTTPHandler) mock_client.post = AsyncMock(return_value=mock_response) result = await litellm.acreate_realtime_client_secret( - model="azure/gpt-realtime-2.1", - session={"type": "realtime", "model": "gpt-realtime-2.1"}, + session={"type": "realtime", "model": "azure/gpt-realtime-2.1"}, api_base="https://my.openai.azure.com", api_key="azure-test-key", api_version="v1", diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index 8513a59cfe2..9a75f1f442d 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -455,7 +455,7 @@ def test_managed_realtime_requires_a_model_and_ignores_completion_defaults(route with pytest.raises(HTTPException, match="explicit or configured model"): managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli") assert ( - managed_inference_request(route, {"model": "requested"}, {"completion_model": "allowed-default"}, "cli")[ + managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli", query_model="requested")[ "model" ] == "requested" @@ -492,19 +492,6 @@ def test_managed_translation_authorizes_the_same_model_as_the_endpoint(route: st assert isinstance(admitted["model"], str) and admitted["model"] -@pytest.mark.parametrize("route", ("/realtime", "/v1/realtime", "/openai/v1/realtime")) -def test_managed_transcription_authorizes_the_same_model_as_the_endpoint(route: str) -> None: - from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request - from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model - - admitted: Final = managed_inference_request( - route, {}, {"completion_model": "unrelated-default"}, "cli", intent="transcription" - ) - - assert admitted["model"] == resolve_realtime_route_model(None, "transcription", is_translation=False) - assert isinstance(admitted["model"], str) and admitted["model"] - - @pytest.mark.parametrize("mode,user", [("autonomous", None), ("delegated", "verified-human")]) def test_matching_identity_revision_and_execution_mode_pass_admission(mode: str, user: str | None) -> None: context: Final = ManagedAgentContext.model_validate( diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 6ea5693bbe6..df2c6aa71af 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -11989,6 +11989,47 @@ async def test_translation_websocket_auth_enforces_the_default_model_budget( assert "budget" in str(denied.value.detail).lower() +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime", "/v1/realtime", "/openai/v1/realtime")) +async def test_managed_agent_transcription_websocket_without_a_model_is_refused( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.types.agents import AgentResponse + + token: Final = UserAPIKeyAuth(token="test-key") + token.managed_agent_policy = AgentResponse.model_validate( + {"agent_id": "agent", "agent_name": "Agent", "agent_card_params": {}, "execution_mode": "both"} + ) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = {**_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), "prisma_client": None} + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": b"intent=transcription", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + with pytest.raises(HTTPException) as denied: + await user_api_key_auth_websocket(websocket) + + assert denied.value.status_code == 403 + assert "Managed inference requires an explicit or configured model" in str(denied.value.detail) + + @pytest.mark.asyncio async def test_admission_and_budget_reservation_read_the_key_spend_counter_with_one_redis_mget(): from fastapi import Request diff --git a/tests/unit/proxy/common_utils/test_realtime_cache.py b/tests/unit/proxy/common_utils/test_realtime_cache.py index 10dc436f274..dfeed94337a 100644 --- a/tests/unit/proxy/common_utils/test_realtime_cache.py +++ b/tests/unit/proxy/common_utils/test_realtime_cache.py @@ -42,20 +42,6 @@ def test_realtime_request_body_caches_each_model_separately(): assert gpt4o_body_first is not gpt4o_mini_body -@pytest.mark.parametrize("is_translation", (False, True)) -def test_realtime_request_body_preserves_default_model_authorization(is_translation: bool) -> None: - from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request - from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model - - body: Final = TypeAdapter(Mapping[str, object]).validate_json(realtime_request_body(None)) - route: Final = "/v1/realtime/translations" if is_translation else "/v1/realtime" - intent: Final = None if is_translation else "transcription" - admitted: Final = managed_inference_request(route, body, {}, None, intent=intent) - - assert admitted["model"] == resolve_realtime_route_model(None, intent, is_translation) - assert isinstance(admitted["model"], str) and admitted["model"] - - @pytest.mark.parametrize("model", (None, "requested", 'model"with\\quotes', "model\nwith-newline")) def test_realtime_request_body_preserves_model_names(model: str | None) -> None: body: Final = TypeAdapter(Mapping[str, object]).validate_json(realtime_request_body(model)) diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index 7f186737f27..d16f5dd8a5d 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -17,15 +17,18 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import respx from fastapi.testclient import TestClient from openai import AsyncOpenAI import litellm from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server +from litellm.proxy.route_llm_request import route_request from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import HttpxBinaryResponseContent from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep +from litellm.types.utils import TranscriptionResponse @pytest.fixture @@ -317,11 +320,8 @@ def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcr @pytest.mark.usefixtures("patched_transcription_stream") -@pytest.mark.parametrize( - "configuration,expected_status", - [("default", 400), ("model", 400), ("policy", 400), ("pre_call", 200), ("disabled", 200)], -) -def test_streaming_transcription_rejects_applicable_output_guardrails( +@pytest.mark.parametrize("configuration", ["default", "policy"]) +def test_streaming_transcription_keeps_streaming_with_output_guardrails( client: TestClient, auth_as: Callable[[], AbstractContextManager[None]], monkeypatch: pytest.MonkeyPatch, @@ -334,9 +334,6 @@ def test_streaming_transcription_rejects_applicable_output_guardrails( default_on=configuration in ("default", "pre_call"), ) monkeypatch.setattr(litellm, "callbacks", [guardrail]) - proxy_server.llm_router.get_model_list.return_value = ( - [{"litellm_params": {"guardrails": ["transcription-output"]}}] if configuration == "model" else [] - ) if configuration == "policy": pipeline: Final = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="transcription-output")]) proxy_server.proxy_logging_obj.pre_call_hook.side_effect = lambda **kwargs: { @@ -406,3 +403,75 @@ def test_streamed_transcription_returns_the_litellm_call_and_model_headers( proxy_server.proxy_logging_obj.update_request_status.assert_called_once_with( litellm_call_id="streamed-transcription-call", status="success" ) + + +class _TranscriptRedactor(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call) + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + if isinstance(response, TranscriptionResponse): + response.text = "[REDACTED]" + + +def _streams_when_asked(request: httpx.Request) -> httpx.Response: + if b'name="stream"' in request.content: + return httpx.Response( + 200, + content=b'data: {"type":"transcript.text.done","text":"my pin is 1234"}\n\n', + headers={"content-type": "text/event-stream"}, + ) + return httpx.Response(200, json={"text": "my pin is 1234"}) + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize("deployment_guardrails", (["transcript-redactor"], None)) +def test_streamed_transcription_honors_a_deployment_post_call_guardrail( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + deployment_guardrails: list[str] | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(side_effect=_streams_when_asked) + deployment_params: Final = { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + } + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": ( + {**deployment_params, "guardrails": deployment_guardrails} + if deployment_guardrails + else deployment_params + ), + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + if deployment_guardrails: + assert response.headers["content-type"].startswith("application/json") + assert response.json()["text"] == "[REDACTED]" + else: + assert response.headers["content-type"].startswith("text/event-stream") + assert '"text":"my pin is 1234"' in response.text diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 387e09a7a70..645d94fe142 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -4,18 +4,21 @@ Tests for LiteLLM proxy realtime WebRTC HTTP endpoints: - POST /v1/realtime/calls """ +import functools import json import time from collections.abc import Awaitable -from typing import Protocol +from typing import Final, Protocol from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient from starlette.routing import Match +import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import ConfigGeneralSettings, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -1607,3 +1610,100 @@ def test_transcription_sessions_rejection_answers_an_openai_typed_error( assert response.status_code == 400 assert (response.json()["error"]["type"], response.json()["error"]["param"]) == ("invalid_request_error", None) + + +_AZURE_REALTIME_DEPLOYMENTS: Final = ( + ("azure-realtime-alias", "azure/qa-realtime-deployment"), + ("azure-transcribe-alias", "azure/qa-whisper-deployment"), + ("azure-translate-alias", "azure/qa-translate-deployment"), +) + + +@pytest.mark.parametrize( + ("path", "body", "upstream_model_path", "expected_model"), + ( + ( + "/v1/realtime/client_secrets", + {"session": {"type": "realtime", "model": "azure-realtime-alias"}}, + ("model",), + "qa-realtime-deployment", + ), + ( + "/v1/realtime/client_secrets", + {"model": "azure-realtime-alias"}, + ("model",), + "qa-realtime-deployment", + ), + ( + "/v1/realtime/client_secrets", + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}}, + } + }, + ("audio", "input", "transcription", "model"), + "qa-whisper-deployment", + ), + ( + "/v1/realtime/translations/client_secrets", + {"model": "azure-translate-alias"}, + ("model",), + "qa-translate-deployment", + ), + ), +) +def test_client_secrets_mint_a_secret_for_the_azure_deployment_model( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: dict[str, object], + upstream_model_path: tuple[str, ...], + expected_model: str, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_list: Final = [ # mutable-ok: Router takes a list of deployments + { + "model_name": alias, + "litellm_params": { + "model": deployment, + "api_base": "https://realtime-qa.openai.azure.com", + "api_key": "azure-test-key", + }, + } + for alias, deployment in _AZURE_REALTIME_DEPLOYMENTS + ] + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + upstream: Final = respx_mock.route(host="realtime-qa.openai.azure.com").mock( + return_value=httpx.Response(200, json={"value": "ek_azure", "expires_at": int(time.time()) + 3600}) + ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", models=[alias for alias, _ in _AZURE_REALTIME_DEPLOYMENTS] + ) + try: + with ( + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response: Final = TestClient(proxy_app).post( + path, headers={"Authorization": "Bearer sk-test-master-key"}, json=body + ) + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200, response.text + upstream_session: Final = json.loads(upstream.calls.last.request.content)["session"] + upstream_model: Final = functools.reduce(lambda node, key: node[key], upstream_model_path, upstream_session) + assert upstream_model == expected_model diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 78adb46698e..5d5e0c285e1 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -19,6 +19,7 @@ from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID +from openai import AsyncOpenAI import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -29,7 +30,7 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai.workload_identity import _workload_identity_auth from litellm.models.credentials import CredentialItem from litellm.realtime_api import main as realtime_main -from litellm.realtime_api.main import _with_resolved_session_model +from litellm.realtime_api.main import with_resolved_session_model from litellm.types.guardrails import GuardrailEventHooks from litellm.types.realtime import RealtimeQueryParams @@ -57,19 +58,19 @@ class _CallCapture: def test_resolves_top_level_session_model(): - resolved = _with_resolved_session_model({"model": "alias/gpt-realtime"}, "gpt-realtime") + resolved = with_resolved_session_model({"model": "alias/gpt-realtime"}, "gpt-realtime") assert resolved == {"model": "gpt-realtime"} def test_session_without_model_is_returned_unchanged(): session = {"type": "realtime", "audio": {"input": {}}} - assert _with_resolved_session_model(session, "gpt-realtime") == session + assert with_resolved_session_model(session, "gpt-realtime") == session def test_does_not_clobber_flat_transcription_model(): """The nested transcription model is a different model than the realtime conversation model and must not be overwritten with the routing model.""" - resolved = _with_resolved_session_model( + resolved = with_resolved_session_model( {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "whisper-1"}}, "gpt-4o-realtime-preview", ) @@ -77,7 +78,7 @@ def test_does_not_clobber_flat_transcription_model(): def test_does_not_clobber_nested_audio_transcription_model(): - resolved = _with_resolved_session_model( + resolved = with_resolved_session_model( { "model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "whisper-1"}}}, @@ -89,7 +90,7 @@ def test_does_not_clobber_nested_audio_transcription_model(): def test_original_session_is_not_mutated(): session = {"model": "alias/gpt-realtime"} - _with_resolved_session_model(session, "gpt-realtime") + with_resolved_session_model(session, "gpt-realtime") assert session == {"model": "alias/gpt-realtime"} @@ -397,6 +398,52 @@ async def test_arealtime_live_session_rejects_streaming_only_pre_call_guardrail( assert live_handler.called is False +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_metadata", + ( + {"user_api_key_metadata": {"disable_global_guardrails": True}}, + {"user_api_key_team_metadata": {"opted_out_global_guardrails": ["default-on-blocker"]}}, + ), +) +async def test_arealtime_key_metadata_cannot_opt_out_of_a_default_on_realtime_guardrail( + monkeypatch: pytest.MonkeyPatch, key_metadata: dict[str, object] +) -> None: + class FakeLiveHandler: + def __init__(self) -> None: + self.called = False + + async def async_live_session(self, **kwargs: object) -> None: + self.called = True + + live_handler: Final = FakeLiveHandler() + monkeypatch.setattr(realtime_main, "openai_live_sessions", live_handler) + monkeypatch.setattr( + litellm, + "callbacks", + [ + CustomGuardrail( + guardrail_name="default-on-blocker", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + ) + ], + ) + + with pytest.raises(ValueError, match="Guardrails are not supported on OpenAI Live sessions"): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-live-1", + websocket=MagicMock(), + api_key="fake-key", + api_base="https://live-gateway.example/v1", + litellm_logging_obj=FakeLogging(), + metadata=key_metadata, + live_session_start={"type": "session.start", "session": {"model": "gpt-live-1"}}, + ) + + assert live_handler.called is False + + @pytest.mark.asyncio async def test_arealtime_live_session_rejects_non_openai_provider(monkeypatch: pytest.MonkeyPatch) -> None: def mock_get_llm_provider( @@ -925,6 +972,30 @@ async def test_arealtime_openai_forwards_the_intent_query_param_to_the_upstream_ assert connect.url == "wss://api.openai.com/v1/realtime?model=gpt-realtime&intent=chat" +@pytest.mark.asyncio +@pytest.mark.parametrize("intent", (None, "transcription")) +async def test_arealtime_openai_connects_to_api_base_rather_than_the_callers_client_base_url( + intent: str | None, +) -> None: + connect: Final = _ConnectThatStopsAfterCapturingTheUrl() + query_params: Final[RealtimeQueryParams] = ( + {"model": "openai/gateway-realtime", "intent": intent} if intent else {"model": "openai/gateway-realtime"} + ) + with patch("websockets.connect", connect), patch("websockets.asyncio.client.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gateway-realtime", + websocket=_ClosableGaClientWebSocket(), + api_key="fake-key", + api_base="https://configured-gateway.example/", + client=AsyncOpenAI(api_key="caller-key", base_url="https://caller-client.example/v1"), + query_params=query_params, + litellm_logging_obj=FakeLogging(), + ) + + expected_query: Final = "model=gateway-realtime&intent=transcription" if intent else "model=gateway-realtime" + assert connect.url == f"wss://configured-gateway.example/v1/realtime?{expected_query}" + + @pytest.mark.parametrize( ("model", "api_base", "client_query_params", "expected_backend_url"), [ @@ -1527,6 +1598,27 @@ async def test_client_secret_keeps_the_realtime_session_model_when_input_transcr assert logging_obj.model == "gpt-realtime" +@pytest.mark.asyncio +async def test_client_secret_session_model_wins_over_a_provider_qualified_top_level_model( + local_model_cost_map: None, +) -> None: + gateway: Final = _GatewayThatRecordsTheSession() + logging_obj: Final = _realtime_http_logging("client_secrets") + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + model="openai/gpt-realtime-2.1", + session={"type": "realtime", "model": "gpt-realtime-2.1-mini"}, + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=logging_obj, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-realtime-2.1-mini" + assert logging_obj.model == "gpt-realtime-2.1-mini" + + class _UpstreamThatMustNotBeCalled: called: bool = False diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index d1d4a1ae591..7445070bd3a 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -673,8 +673,6 @@ def test_every_openai_sdk_response_error_code_has_explicit_status_mapping(): ("insufficient_quota", 429), ("vector_store_timeout", 504), ("invalid_prompt", 400), - ("data_residency_mismatch", 400), - ("bio_policy", 400), ("invalid_image", 400), ("invalid_image_format", 400), ("invalid_base64_image", 400), diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index c3d3575b8b3..4a2899c4ef1 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5070,6 +5070,42 @@ async def test_groq_atranscription_honors_base_url_alias(respx_mock: respx.MockR assert response.text == "hello" +@pytest.mark.parametrize("model_name", ("whisper-1", "gpt-4o-transcribe", "gpt-transcribe")) +def test_openai_compatible_transcription_with_an_openai_model_name_reaches_the_server( + respx_mock: respx.MockRouter, model_name: str +): + route: Final = respx_mock.post("https://vllm.internal/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "served by vllm"}) + ) + + response: Final = litellm.transcription( + model=f"hosted_vllm/{model_name}", + file=GROQ_WAV_FILE, + api_base="https://vllm.internal/v1", + api_key="fake-key", + response_format="verbose_json", + ) + + assert route.called + assert response.text == "served by vllm" + + +def test_openai_gpt_transcribe_still_rejects_non_json_response_formats(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.openai.com/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "should not be sent"}) + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"): + litellm.transcription( + model="openai/gpt-transcribe", + file=GROQ_WAV_FILE, + api_key="fake-key", + response_format="srt", + ) + + assert not route.called + + def test_groq_speech_honors_base_url_alias(respx_mock: respx.MockRouter): route: Final = respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/speech").mock( return_value=httpx.Response(200, content=b"mp3-bytes") From 34319dff7df1b056d4e7dca5ac4d1de7f24b198a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:12:28 -0700 Subject: [PATCH 41/45] fix(realtime): keep main's transcription, client secret and route behavior where the PR changed it Transcription reaches every provider as on main again, sends stream only when asked, forwards language and languages together, keeps form key order, bills provider seconds only when above zero, honors base_model for the new flags, keeps main's Azure api_version on the proxy path, and streams carry chat's headers, an error event on a mid-stream failure, and a spend row when the stream ends early. Translation cost reads both per-second rates from one source and spend rows carry the billed seconds. Realtime transcription WebSockets keep model in the upstream URL outside OpenAI's hosts, client secrets forward explicit nulls, accept conversation and delay, and answer 400 on a malformed body, the /openai translation paths are back to main, translation refuses only for guardrails realtime sessions run, nested aliases resolve and later session.update models are authorized, translation client secrets run per-model auth, beta clients bridged to GA get beta-shaped session events, and transcription sessions send the deployment model in both shapes. --- litellm/cost_calculator.py | 71 ++--- .../audio_utils/transcription_streaming.py | 35 +-- .../convert_dict_to_response.py | 9 +- .../litellm_core_utils/realtime_streaming.py | 154 +++++++--- litellm/llms/azure/audio_transcriptions.py | 13 +- .../audio_transcription/transformation.py | 58 ++++ litellm/main.py | 36 +-- litellm/proxy/_lazy_openapi_snapshot.json | 24 -- litellm/proxy/_types.py | 3 - .../auth/managed_authorization.py | 1 - litellm/proxy/auth/auth_utils.py | 1 + .../proxy/common_utils/http_parsing_utils.py | 2 +- litellm/proxy/proxy_server.py | 62 ++-- litellm/proxy/realtime_endpoints/endpoints.py | 33 +- litellm/realtime_api/main.py | 62 ++-- litellm/types/realtime.py | 7 +- litellm/types/utils.py | 21 +- litellm/utils.py | 3 +- scripts/check_openapi_docs.py | 6 +- .../test_realtime_streaming.py | 283 ++++++++++-------- .../llms/azure/test_audio_transcriptions.py | 45 ++- .../transcriptions/test_gpt_transcribe.py | 257 ++++++++++++---- .../auth/test_managed_authorization.py | 4 +- tests/unit/proxy/auth/test_route_checks.py | 21 +- .../test_user_api_key_auth_request_flow.py | 50 +++- .../common_utils/test_http_parsing_utils.py | 11 + .../proxy/proxy_server/test_routes_audio.py | 114 ++++++- .../test_realtime_webrtc_endpoints.py | 209 +++++++++---- tests/unit/proxy/test_proxy_server.py | 86 ++++++ ...test_proxy_server_endpoints_and_startup.py | 3 +- tests/unit/realtime_api/test_main.py | 119 +++++++- tests/unit/test_check_openapi_docs.py | 19 +- tests/unit/test_cost_calculator.py | 86 ++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 58 +--- 34 files changed, 1427 insertions(+), 539 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 09a7c2a61e4..51251718983 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -352,22 +352,24 @@ def _transcription_uses_token_pricing( return (prompt_tokens_val > 0) or (completion_tokens_val > 0) -def _transcription_duration_for_cost( - completion_response: object, model: str | None, custom_llm_provider: str | None -) -> float: - provider_duration: Final = _get_transcription_usage_duration(completion_response) - transcription_info: Final = ( - _get_model_info_or_none(model, custom_llm_provider) - if model is not None and custom_llm_provider is not None - else None - ) +def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool: + if pricing_model is None or custom_llm_provider is None: + return False + transcription_info: Final = _get_model_info_or_none(pricing_model, custom_llm_provider) transcription_metadata: Final = ( as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None ) + return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1 + + +def _transcription_duration_for_cost( + completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None +) -> float: + provider_duration: Final = _get_transcription_usage_duration(completion_response) if ( provider_duration is not None - and transcription_metadata is not None - and transcription_metadata.get("transcription_usage_duration") == 1 + and provider_duration > 0 + and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model)) ): return provider_duration hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {} @@ -1907,7 +1909,7 @@ def completion_cost( prompt_characters = litellm.utils.count_characters(text=prompt) elif call_type in _TRANSCRIPTION_CALL_TYPES: audio_transcription_file_duration = _transcription_duration_for_cost( - completion_response, requested_model, custom_llm_provider + completion_response, model, requested_model, custom_llm_provider ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): @@ -3213,8 +3215,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor): def create_logging_realtime_object( usage: Usage, results: OpenAIRealtimeStreamList ) -> LiteLLMRealtimeStreamLoggingObject: + _, translated_output_seconds = _translation_session_seconds(results) return LiteLLMRealtimeStreamLoggingObject( - usage=usage, + usage=usage.model_copy(update={"seconds": translated_output_seconds}) + if translated_output_seconds > 0 + else usage, results=results, ) @@ -3661,12 +3666,7 @@ def handle_realtime_stream_cost_calculation( return total_cost -def handle_realtime_translation_cost_calculation( - results: OpenAIRealtimeStreamList, - custom_llm_provider: str, - litellm_model_name: str, - potential_model_names: Sequence[str | None] = (), -) -> float: +def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]: usage_events: Final = tuple( usage for result in results @@ -3684,6 +3684,19 @@ def handle_realtime_translation_cost_calculation( for usage in usage_events if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None ) + return float(input_seconds), float(output_seconds) + + +_TRANSLATION_RATE_KEYS: Final = ("input_cost_per_second", "output_cost_per_second") + + +def handle_realtime_translation_cost_calculation( + results: OpenAIRealtimeStreamList, + custom_llm_provider: str, + litellm_model_name: str, + potential_model_names: Sequence[str | None] = (), +) -> float: + input_seconds, output_seconds = _translation_session_seconds(results) if input_seconds <= 0 and output_seconds <= 0: return 0.0 model_infos: Final = tuple( @@ -3691,22 +3704,12 @@ def handle_realtime_translation_cost_calculation( for model in (*potential_model_names, litellm_model_name) if model is not None ) - input_cost_per_second: Final = next( - ( - rate - for info in model_infos - if (rate := _declared_transcription_rate(info, ("input_cost_per_second",))) is not None - ), - 0.0, - ) - output_cost_per_second: Final = next( - ( - output_rate - for info in model_infos - if (output_rate := _declared_transcription_rate(info, ("output_cost_per_second",))) is not None - ), - 0.0, + priced_info: Final = next( + (info for info in model_infos if _declared_transcription_rate(info, _TRANSLATION_RATE_KEYS) is not None), + None, ) + input_cost_per_second: Final = _declared_transcription_rate(priced_info, ("input_cost_per_second",)) or 0.0 + output_cost_per_second: Final = _declared_transcription_rate(priced_info, ("output_cost_per_second",)) or 0.0 return input_seconds * input_cost_per_second + output_seconds * output_cost_per_second diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 0dd18fe581b..1fc18d9eda0 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -2,7 +2,7 @@ from __future__ import annotations import datetime import traceback -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator from typing import Final, Protocol from openai import AsyncStream, Stream @@ -11,11 +11,9 @@ from openai.types.audio import ( TranscriptionTextDeltaEvent, TranscriptionTextDoneEvent, ) -from pydantic import TypeAdapter -from litellm.types.utils import TranscriptionDetectedLanguage, TranscriptionResponse - -_DETECTED_LANGUAGES: Final = TypeAdapter(list[TranscriptionDetectedLanguage]) +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.types.utils import TranscriptionResponse class TranscriptionStreamLogging(Protocol): @@ -71,15 +69,10 @@ class _TranscriptionEventCollector: def response(self) -> TranscriptionResponse: done_event: Final = self.done_event - done_languages: Final = getattr(done_event, "languages", None) if done_event is not None else None response: Final = TranscriptionResponse( text=done_event.text if done_event is not None else "".join(self.text_deltas), usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, - languages=( - _DETECTED_LANGUAGES.validate_python(done_languages, from_attributes=True) - if done_languages is not None - else None - ), + languages=getattr(done_event, "languages", None) if done_event is not None else None, ) if self.duration is not None: response.set_audio_transcription_duration(self.duration) @@ -94,7 +87,9 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): start_time: datetime.datetime, ) -> None: self.__dict__.update(stream.__dict__) - self._hidden_params: Mapping[str, object] = {} + self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata + "additional_headers": process_response_headers(stream.response.headers) + } self._logging_obj = logging_obj self._start_time = start_time self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) @@ -122,8 +117,6 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): if self._finalized or self._failed: return self._finalized = True - if self._collector.done_event is None and self._collector.duration is None: - return self._logging_obj.success_handler( self._collector.response(), self._start_time, @@ -136,6 +129,10 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): finally: self._finalize() + @property + def logging_obj(self) -> TranscriptionStreamLogging: + return self._logging_obj + class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): def __init__( @@ -145,7 +142,9 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): start_time: datetime.datetime, ) -> None: self.__dict__.update(stream.__dict__) - self._hidden_params: Mapping[str, object] = {} + self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata + "additional_headers": process_response_headers(stream.response.headers) + } self._logging_obj = logging_obj self._start_time = start_time self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) @@ -179,8 +178,6 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): if self._finalized or self._failed: return self._finalized = True - if self._collector.done_event is None and self._collector.duration is None: - return response: Final = self._collector.response() end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract self._logging_obj.handle_sync_success_callbacks_for_async_calls( @@ -200,6 +197,10 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): finally: await self._finalize() + @property + def logging_obj(self) -> TranscriptionStreamLogging: + return self._logging_obj + def wrap_transcription_stream( stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent], diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index e49526ad505..71d5be0a417 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -44,7 +44,6 @@ from litellm.types.utils import ( StreamingChoices, TextChoices, TextCompletionResponse, - TranscriptionDetectedLanguage, TranscriptionResponse, TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, @@ -947,11 +946,9 @@ def convert_to_model_response_object( if key in response_object: setattr(model_response_object, key, response_object[key]) - if "languages" in response_object and response_object["languages"] is not None: - transcription_response: Final = model_response_object - transcription_response.languages = TypeAdapter( - tuple[TranscriptionDetectedLanguage, ...] - ).validate_python(response_object["languages"]) + detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages")) + if detected_languages is not None: + model_response_object.languages = detected_languages if "usage" in response_object and response_object["usage"] is not None: tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 851fbdd51fb..8aa985ed507 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -43,6 +43,7 @@ if TYPE_CHECKING: from websockets.asyncio.client import ClientConnection from websockets.exceptions import ConnectionClosed + from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks CLIENT_CONNECTION_CLASS = ClientConnection @@ -151,6 +152,52 @@ def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, ob return _json_object(container.get(key)) +_BETA_AUDIO_FORMAT_BY_GA_TYPE: Final[Mapping[str, str]] = MappingProxyType( + {"audio/pcm": "pcm16", "audio/pcmu": "g711_ulaw", "audio/pcma": "g711_alaw"} +) +_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"output_modalities": "modalities", "max_output_tokens": "max_response_output_tokens"} +) +_GA_TO_BETA_AUDIO_INPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + { + "format": "input_audio_format", + "transcription": "input_audio_transcription", + "noise_reduction": "input_audio_noise_reduction", + "turn_detection": "turn_detection", + } +) +_GA_TO_BETA_AUDIO_OUTPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"format": "output_audio_format", "voice": "voice", "speed": "speed"} +) +_GA_ONLY_SESSION_FIELDS: Final = frozenset(("type", "audio", *_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS)) + + +def _beta_session_value(ga_field: str, value: object) -> object: + if ga_field == "output_modalities": + return ["audio", "text"] if value == ["audio"] else value + if ga_field != "format": + return value + ga_format: Final = _json_object(value) + format_type: Final = None if ga_format is None else ga_format.get("type") + return _BETA_AUDIO_FORMAT_BY_GA_TYPE.get(format_type, value) if isinstance(format_type, str) else value + + +def _beta_session_fields(ga_fields: Mapping[str, object], beta_names: Mapping[str, str]) -> Mapping[str, object]: + return { + beta_names[name]: _beta_session_value(name, value) for name, value in ga_fields.items() if name in beta_names + } + + +def ga_session_to_beta(session: Mapping[str, object]) -> Mapping[str, object]: + audio: Final = _mapping_field(session, "audio") or _EMPTY_SESSION + return { + **{key: value for key, value in session.items() if key not in _GA_ONLY_SESSION_FIELDS}, + **_beta_session_fields(session, _GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS), + **_beta_session_fields(_mapping_field(audio, "input") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_INPUT_FIELDS), + **_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS), + } + + def _guardrail_event_id() -> str: return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}" @@ -265,26 +312,29 @@ def _guardrails_run_for_event_hooks( ) +def _realtime_runner_guardrail( + callback: object, + check_data: Mapping[str, object], + event_hooks: Sequence["GuardrailEventHooks"], +) -> "CustomGuardrail | None": + from litellm.integrations.custom_guardrail import CustomGuardrail + + if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks: + return None + runs: Final = any( + callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks + ) + return callback if runs else None + + def realtime_guardrails_apply(request_data: Mapping[str, object]) -> bool: from litellm.types.guardrails import GuardrailEventHooks - if any( - isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket - for bucket in (request_data.get("metadata"), request_data.get("litellm_metadata")) - ): - from litellm.proxy.utils import pipeline_managed_guardrail_names - - if pipeline_managed_guardrail_names(request_data, "pre_call") or pipeline_managed_guardrail_names( - request_data, "post_call" - ): - return True - return _guardrails_run_for_event_hooks( - request_data, - ( - GuardrailEventHooks.realtime_input_transcription, - GuardrailEventHooks.pre_call, - GuardrailEventHooks.post_call, - ), + check_data: Final = {**request_data, "stream": True} + runner_event_hooks: Final = (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call) + return any( + _realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None + for callback in litellm.callbacks ) @@ -933,40 +983,43 @@ class RealTimeStreaming: return self._rewrite_session_update_transcription_model(message, self._force_transcription_model) async def _enforce_translation_nested_transcription_model(self, message: str) -> str: - if self._bound_nested_transcription_model is not None: - return self._rewrite_session_update_transcription_model(message, self._bound_nested_transcription_model) - message_obj: Final = self._session_update_message_obj(message) - if message_obj is None: - return message - session: Final = as_str_mapping(message_obj.get("session")) - if session is None: - return message - nested_models: Final = self._nested_transcription_models_from_session(session) - if not nested_models: - return message + session: Final = as_str_mapping(message_obj.get("session")) if message_obj is not None else None + bound_model: Final = self._bound_nested_transcription_model + requested_models: Final = ( + tuple(model for model in self._nested_transcription_models_from_session(session) if model != bound_model) + if session is not None + else () + ) + if not requested_models: + return ( + message + if bound_model is None + else self._rewrite_session_update_transcription_model(message, bound_model) + ) valid_token: Final = self.user_api_key_dict - if valid_token is None: - return message from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import can_key_call_resolved_model from litellm.proxy.proxy_server import llm_model_list, llm_router + from litellm.proxy.realtime_endpoints.endpoints import nested_transcription_upstream_model if not isinstance(valid_token, UserAPIKeyAuth): return message - for nested_model in nested_models: + for requested_model in requested_models: await can_key_call_resolved_model( - model=nested_model, + model=requested_model, valid_token=valid_token, llm_model_list=llm_model_list, llm_router=llm_router, ) - bound_model: Final = nested_models[0] - self._bound_nested_transcription_model = bound_model - return self._rewrite_session_update_transcription_model(message, bound_model) + upstream_model: Final = nested_transcription_upstream_model( + requested_models[0], valid_token.team_id, llm_router + ) + self._bound_nested_transcription_model = upstream_model + return self._rewrite_session_update_transcription_model(message, upstream_model) def _uses_deferred_backend_setup(self) -> bool: """True when setup is deferred until the client's first session.update.""" @@ -1087,7 +1140,18 @@ class RealTimeStreaming: return event def _event_to_client_json(self, event: dict) -> str: - return json.dumps(self._normalize_event_for_ga_client(event)) + return json.dumps(self._session_event_for_client(self._normalize_event_for_ga_client(event))) + + def _session_event_for_client(self, event: Mapping[str, object]) -> Mapping[str, object]: + if ( + not self._client_wants_beta + or self._backend_uses_beta_protocol + or self._is_translation_session + or event.get("type") not in self._SESSION_EVENT_TYPES + ): + return event + session: Final = _mapping_field(event, "session") + return event if session is None else {**event, "session": ga_session_to_beta(session)} async def _send_event_to_client(self, event: object, event_str: str) -> bool: if self._should_drop_event_from_client(event): @@ -1100,7 +1164,7 @@ class RealTimeStreaming: translated: Final = self._translate_event_to_beta(event) if translated is None: return False - await self.websocket.send_text(json.dumps(translated)) + await self.websocket.send_text(json.dumps(self._session_event_for_client(translated))) return True except Exception as e: verbose_logger.warning( @@ -1253,7 +1317,6 @@ class RealTimeStreaming: transcript completion uses ``realtime_input_transcription`` only; typed user messages and tool outputs use ``pre_call``. """ - from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks if event_hooks is None: @@ -1262,14 +1325,11 @@ class RealTimeStreaming: _check_data: Final = {**self.request_data, "transcript": transcript, "stream": True} _already_run: Final[set] = set() - for callback in litellm.callbacks: - if not isinstance(callback, CustomGuardrail): - continue - if callback.use_native_lifecycle_hooks: - continue - if id(callback) in _already_run: - continue - if not any(callback.should_run_guardrail(data=_check_data, event_type=et) for et in _realtime_event_types): + runner_guardrails: Final = ( + _realtime_runner_guardrail(callback, _check_data, _realtime_event_types) for callback in litellm.callbacks + ) + for callback in runner_guardrails: + if callback is None or id(callback) in _already_run: continue _already_run.add(id(callback)) try: @@ -1547,7 +1607,7 @@ class RealTimeStreaming: translated = self._translate_event_to_beta(event) if translated is None: continue - await self.websocket.send_text(json.dumps(translated)) + await self.websocket.send_text(json.dumps(self._session_event_for_client(translated))) async def backend_to_client_send_messages(self) -> BackendClose: import websockets diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 8f5a26d41a4..d4f53dc0dd5 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,7 +1,7 @@ from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final, cast -from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI, Stream +from openai import AsyncAzureOpenAI, AsyncStream, AzureOpenAI, Stream from openai.types.audio import TranscriptionStreamEvent from openai.types.audio.transcription_create_params import ( TranscriptionCreateParamsBase, @@ -9,7 +9,6 @@ from openai.types.audio.transcription_create_params import ( ) from pydantic import BaseModel -import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.llms.base_llm.audio_transcription.transformation import ( @@ -55,9 +54,6 @@ class AzureAudioTranscription(AzureChatCompletion): ): data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) - resolved_api_version: Final = ( - litellm.AZURE_DEFAULT_API_VERSION if api_version in ("v1", "latest", "preview") else api_version - ) if atranscription is True: return self.async_audio_transcriptions( @@ -67,7 +63,6 @@ class AzureAudioTranscription(AzureChatCompletion): timeout=timeout, api_key=api_key, api_base=api_base, - api_version=resolved_api_version, client=client, max_retries=max_retries, logging_obj=logging_obj, @@ -77,7 +72,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) azure_client: Final = self.get_azure_openai_client( - api_version=resolved_api_version, + api_version=None if self._is_azure_v1_api_version(api_version) else api_version, api_base=api_base, api_key=api_key, model=model, @@ -85,7 +80,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(azure_client, (AzureOpenAI, OpenAI)): + if not isinstance(azure_client, AzureOpenAI): raise AzureOpenAIError( status_code=500, message="azure_client is not an instance of AzureOpenAI", @@ -169,7 +164,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(async_azure_client, (AsyncAzureOpenAI, AsyncOpenAI)): + if not isinstance(async_azure_client, AsyncAzureOpenAI): raise AzureOpenAIError( status_code=500, message="async_azure_client is not an instance of AsyncAzureOpenAI", diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index b2b99d92f3d..872bf447e40 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -50,6 +50,64 @@ def sdk_compatible_transcription_response(response: object) -> BaseModel | str: return _TRANSCRIPTION_TEXT.validate_python(response, strict=True) +def _content_type(http_response: httpx.Response) -> str: + return next(iter(http_response.headers.get_list("content-type")), "") + + +def _is_event_stream(http_response: httpx.Response) -> bool: + return _content_type(http_response).startswith("text/event-stream") + + +def _unstreamed_transcription_response(http_response: httpx.Response) -> BaseModel | str: + if _content_type(http_response).startswith("application/json"): + return Transcription.model_validate_json(http_response.content) + return http_response.text + + +def _non_streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsBase: + return cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"} + ) + + +def _streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsStreaming: + return cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsStreaming, data + ) + + +def create_sdk_transcription( + client: OpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout +) -> tuple[dict[str, str], Stream[TranscriptionStreamEvent] | BaseModel | str]: + if data.get("stream") is not True: + raw_response: Final = client.audio.transcriptions.with_raw_response.create( + **_non_streaming_params(data), timeout=timeout + ) + return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) + raw_stream: Final = client.audio.transcriptions.with_raw_response.create(**_streaming_params(data), timeout=timeout) + if _is_event_stream(raw_stream.http_response): + return dict(raw_stream.headers), raw_stream.parse(to=Stream[TranscriptionStreamEvent]) + raw_stream.http_response.read() + return dict(raw_stream.headers), _unstreamed_transcription_response(raw_stream.http_response) + + +async def acreate_sdk_transcription( + client: AsyncOpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout +) -> tuple[dict[str, str], AsyncStream[TranscriptionStreamEvent] | BaseModel | str]: + if data.get("stream") is not True: + raw_response: Final = await client.audio.transcriptions.with_raw_response.create( + **_non_streaming_params(data), timeout=timeout + ) + return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) + raw_stream: Final = await client.audio.transcriptions.with_raw_response.create( + **_streaming_params(data), timeout=timeout + ) + if _is_event_stream(raw_stream.http_response): + return dict(raw_stream.headers), raw_stream.parse(to=AsyncStream[TranscriptionStreamEvent]) + await raw_stream.http_response.aread() + return dict(raw_stream.headers), _unstreamed_transcription_response(raw_stream.http_response) + + @dataclass class AudioTranscriptionRequestData: """ diff --git a/litellm/main.py b/litellm/main.py index b8e39c2b18f..fef790e7911 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8252,37 +8252,25 @@ async def atranscription( def _validate_gpt_transcription_request( model: str, + pricing_model: str, custom_llm_provider: str, - language: str | None, - languages: Sequence[str] | None, response_format: str | None, drop_params: object, ) -> str | None: model_cost_key: Final = next( ( key - for key in (f"{custom_llm_provider}/{model}", model) + for key in (f"{custom_llm_provider}/{pricing_model}", pricing_model) if (as_str_mapping(litellm.model_cost.get(key)) or {}).get("litellm_provider") == custom_llm_provider ), None, ) model_info: Final = ( - get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_cost_key is not None else None + get_model_info(model=pricing_model, custom_llm_provider=custom_llm_provider) + if model_cost_key is not None + else None ) - supported_endpoints: Final = model_info.get("supported_endpoints") if model_info is not None else None provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None - if language is not None and languages is not None: - raise litellm.UnsupportedParamsError( - message="language and languages cannot be used together", - model=model, - llm_provider=custom_llm_provider, - ) - if supported_endpoints is not None and "/v1/audio/transcriptions" not in supported_endpoints: - raise litellm.UnsupportedParamsError( - message=f"{model} is available through the Realtime API, not file transcription", - model=model, - llm_provider=custom_llm_provider, - ) if ( provider_specific_entry is None or provider_specific_entry.get("transcription_json_or_text_only") != 1 @@ -8363,11 +8351,18 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key + litellm_params_dict: Final = get_litellm_params(**kwargs) + configured_base_model: Final = litellm_params_dict.get("base_model") + pricing_model: Final = ( + configured_base_model.removeprefix(f"{custom_llm_provider}/") + if isinstance(configured_base_model, str) + else model + ) + supported_response_format: Final = _validate_gpt_transcription_request( model=model, + pricing_model=pricing_model, custom_llm_provider=custom_llm_provider, - language=language, - languages=languages, response_format=response_format, drop_params=kwargs.get("drop_params"), ) @@ -8386,8 +8381,6 @@ def transcription( **non_default_params, ) - litellm_params_dict: Final = get_litellm_params(**kwargs) - litellm_logging_obj.update_environment_variables( model=model, user=user, @@ -8415,6 +8408,7 @@ def transcription( provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model, provider=LlmProviders(custom_llm_provider), + base_model=pricing_model, ) uses_openai_transport: Final = custom_llm_provider in OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS and not ( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 7e544cf8aa9..ba9b5a8c1a9 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -53586,13 +53586,6 @@ "delay": { "anyOf": [ { - "enum": [ - "minimal", - "low", - "medium", - "high", - "xhigh" - ], "type": "string" }, { @@ -53877,11 +53870,6 @@ "type": { "anyOf": [ { - "enum": [ - "realtime", - "transcription", - "translation" - ], "type": "string" }, { @@ -54152,13 +54140,6 @@ "delay": { "anyOf": [ { - "enum": [ - "minimal", - "low", - "medium", - "high", - "xhigh" - ], "type": "string" }, { @@ -54443,11 +54424,6 @@ "type": { "anyOf": [ { - "enum": [ - "realtime", - "transcription", - "translation" - ], "type": "string" }, { diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b22886a902a..9f1810226e7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -329,7 +329,6 @@ class LiteLLMRoutes(enum.Enum): "/openai/v1/realtime?{model}", "/realtime/translations", "/v1/realtime/translations", - "/openai/v1/realtime/translations", # realtime (GA WebRTC HTTP routes) "/realtime/client_secrets", "/v1/realtime/client_secrets", @@ -342,10 +341,8 @@ class LiteLLMRoutes(enum.Enum): "/openai/v1/realtime/transcription_sessions", "/realtime/translations/client_secrets", "/v1/realtime/translations/client_secrets", - "/openai/v1/realtime/translations/client_secrets", "/realtime/translations/calls", "/v1/realtime/translations/calls", - "/openai/v1/realtime/translations/calls", # responses API "/responses", "/v1/responses", diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 427ed071dc4..57e99edecba 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -18,7 +18,6 @@ _MANAGED_REALTIME_ROUTES: Final = frozenset( "/openai/v1/realtime", "/realtime/translations", "/v1/realtime/translations", - "/openai/v1/realtime/translations", ) ) _MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions")) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 7a186d84aa5..68d6392e381 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1958,6 +1958,7 @@ _MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS: Final = ("/evals",) # can_key_call_model() validates the real target model. _MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS: Final = ( "/realtime/client_secrets", + "/realtime/translations/client_secrets", "/realtime/calls", ) _MODEL_ROUTING_ID_FIELDS: Final = ( diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 796ce3333f8..9f3cee6e2be 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -536,7 +536,7 @@ async def get_form_data(request: Request) -> dict[str, Any]: form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items()) array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]")) normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items) - normalized_keys: Final = frozenset(key for key, _ in normalized_items) + normalized_keys: Final = tuple(dict.fromkeys(key for key, _ in normalized_items)) return { key: [value for item_key, value in normalized_items if item_key == key] if key in array_keys diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 53ce7652256..c80c02142a3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10382,6 +10382,23 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob return _resolve +def _sse_error_frame(client_error: Exception) -> str: + error_returned: Final = json.dumps( + { + "error": sse_error_payload(client_error)[1] + if isinstance(client_error, HTTPException) + else ProxyException( + message=getattr(client_error, "message", str(client_error)), + type=getattr(client_error, "type", "None"), + param=getattr(client_error, "param", "None"), + code=getattr(client_error, "status_code", 500), + ).to_dict() + }, + default=str, + ) + return f"data: {error_returned}\n\n" + + async def async_data_generator( response, user_api_key_dict: UserAPIKeyAuth, @@ -10623,21 +10640,8 @@ async def async_data_generator( return if isinstance(e, HTTPException) and (not stream_started or not ends_with_openai_done): raise client_error - error_returned: Final = json.dumps( - { - "error": sse_error_payload(client_error)[1] - if isinstance(client_error, HTTPException) - else ProxyException( - message=getattr(client_error, "message", str(client_error)), - type=getattr(client_error, "type", "None"), - param=getattr(client_error, "param", "None"), - code=getattr(client_error, "status_code", 500), - ).to_dict() - }, - default=str, - ) stream_completed = True - yield f"data: {error_returned}\n\n" + yield _sse_error_frame(client_error) if ends_with_openai_done: yield _OPENAI_STREAM_DONE_FRAME finally: @@ -13466,6 +13470,7 @@ async def audio_transcriptions( response_cost: Final = hidden_params.get("response_cost", None) or "" response_call_id: Final = hidden_params.get("litellm_call_id", None) or litellm_call_id additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {} + stream_logging_obj: Final = getattr(response, "logging_obj", None) custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=user_api_key_dict, @@ -13478,6 +13483,7 @@ async def audio_transcriptions( call_id=response_call_id, request_data=data, hidden_params=hidden_params, + litellm_logging_obj=stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None, **additional_headers, ) @@ -13499,6 +13505,13 @@ async def audio_transcriptions( try: async for event in stream: yield f"data: {event.model_dump_json()}\n\n" + except Exception as stream_error: # noqa: BLE001 # stream boundary: a mid-stream failure ends the SSE body as an error event + replacement: Final = await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=stream_error, + request_data=data, + ) + yield _sse_error_frame(stream_error if replacement is None else replacement) finally: close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None) if callable(close): @@ -13659,23 +13672,9 @@ async def _reject_realtime_session( await _release_realtime_max_parallel_slot(user_api_key_dict) -def _resolve_realtime_upstream_query_model( - model: str | None, - intent: str | None, - is_translation: bool, - route_model: str, -) -> str | None: - if intent == "transcription": - return None - if is_translation: - return route_model - return model - - @app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") -@app.websocket("/openai/v1/realtime/translations") @app.websocket("/v1/realtime/translations") @app.websocket("/realtime/translations") async def realtime_websocket_endpoint( @@ -13738,12 +13737,7 @@ async def _route_realtime_websocket_session( is_translation: bool = False, ) -> None: # Only use explicit parameters, not all query params - query_model: Final = _resolve_realtime_upstream_query_model( - model=model, - intent=intent, - is_translation=is_translation, - route_model=route_model, - ) + query_model: Final = route_model if is_translation else model query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys RealtimeQueryParams, dict(_realtime_query_params_template(query_model, intent)), diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 5471cfc09af..1e57add6b10 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -2,14 +2,17 @@ import json import time -from collections.abc import Mapping +from collections.abc import Generator, Mapping +from contextlib import contextmanager from typing import TYPE_CHECKING, Any, Final import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi import status as http_status from fastapi.responses import PlainTextResponse +from pydantic import ValidationError +import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import can_key_call_resolved_model @@ -51,6 +54,14 @@ _NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE: Final = ( ) +@contextmanager +def _invalid_request_body_answers_400() -> Generator[None, None, None]: + try: + yield + except ValidationError as error: + raise ProxyException(message=str(error), type="invalid_request_error", param=None, code=400) from error + + _TRANSLATION_SECRET_BODY: Final = { "requestBody": inline_request_body( RealtimeClientSecretRequest, @@ -151,6 +162,16 @@ def _set_transcription_model_on_session( } +def nested_transcription_upstream_model(model: str, team_id: str | None, llm_router: "Router | None") -> str: + deployment_model: Final = llm_router.get_routable_upstream_model(model, team_id) if llm_router is not None else None + if deployment_model is None: + return model + try: + return litellm.get_llm_provider(model=deployment_model)[0] + except litellm.BadRequestError: + return deployment_model + + async def _authorize_and_bind_nested_transcription_models( session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization user_api_key_dict: UserAPIKeyAuth, @@ -168,7 +189,7 @@ async def _authorize_and_bind_nested_transcription_models( if nested_models: _set_transcription_model_on_session( session=session_data, - model=nested_models[0], + model=nested_transcription_upstream_model(nested_models[0], user_api_key_dict.team_id, llm_router), ) @@ -184,7 +205,7 @@ async def _prepare_client_secret_session( raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint") session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type) session_data: Final[dict | None] = ( - req.session.model_dump(exclude_none=True) if req.session else ({} if session_type == "translation" else None) + req.session.model_dump(exclude_unset=True) if req.session else ({} if session_type == "translation" else None) ) if session_data is not None: session_data["type"] = session_type @@ -334,7 +355,8 @@ async def create_realtime_client_secret( data: dict = {} try: body: Final = await read_request_body(request=request) - req: Final = RealtimeClientSecretRequest(**body) + with _invalid_request_body_answers_400(): + req: Final = RealtimeClientSecretRequest.model_validate(body) is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path model, session_data, session_type = await _prepare_client_secret_session( @@ -683,7 +705,8 @@ async def create_realtime_transcription_session( data: dict = {} try: body: Final = await read_request_body(request=request) - req: Final = RealtimeTranscriptionSessionRequest(**body) + with _invalid_request_body_answers_400(): + req: Final = RealtimeTranscriptionSessionRequest.model_validate(body) model: Final[str] = req.resolved_model() or "gpt-realtime-whisper" await can_key_call_resolved_model( diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index f11714218a8..53d53237773 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -249,6 +249,14 @@ def _get_realtime_http_extra_headers( ) +def _client_secret_session_body(req: RealtimeClientSecretRequest, model_name: str) -> Mapping[str, object] | None: + if req.session is not None: + return with_resolved_session_model(req.session.model_dump(exclude_unset=True), model_name) + if req.model is not None: + return {"type": "realtime", "model": model_name} + return None + + @wrapper_client async def acreate_realtime_client_secret( model: str | None = None, @@ -305,20 +313,16 @@ async def acreate_realtime_client_secret( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - if isinstance(request_data.get("session"), dict): - request_data["session"] = with_resolved_session_model(request_data["session"], model_name) - elif req.model is not None: - request_data["session"] = { - "type": "realtime", - "model": model_name, - } + session_body: Final = _client_secret_session_body(req, model_name) return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params ), - request_data=request_data, + request_data={ + **req.model_dump(exclude_none=True, exclude={"model", "session"}), + **({} if session_body is None else {"session": session_body}), + }, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, provider_config=provider_config, @@ -384,18 +388,16 @@ async def acreate_realtime_translation_client_secret( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump( - exclude_none=True, - exclude={"model"}, - ) - request_data["session"] = with_resolved_session_model(request_data["session"], model_name) - request_data["session"].pop("type", None) + translation_session: Final = with_resolved_session_model(session_config.model_dump(exclude_unset=True), model_name) return await base_llm_http_handler.async_realtime_translation_client_secret_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params ), - request_data=request_data, + request_data={ + **req.model_dump(exclude_none=True, exclude={"model", "session"}), + "session": {key: value for key, value in translation_session.items() if key != "type"}, + }, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, provider_config=provider_config, @@ -406,6 +408,22 @@ async def acreate_realtime_translation_client_secret( ) +def _with_upstream_transcription_model(request_data: Mapping[str, object], model_name: str) -> dict[str, object]: + flat: Final = as_str_mapping(request_data.get("input_audio_transcription")) + audio: Final = as_str_mapping(request_data.get("audio")) or {} + audio_input: Final = as_str_mapping(audio.get("input")) or {} + nested: Final = as_str_mapping(audio_input.get("transcription")) + return { + **request_data, + **({} if flat is None else {"input_audio_transcription": {**flat, "model": model_name}}), + **( + {} + if nested is None + else {"audio": {**audio, "input": {**audio_input, "transcription": {**nested, "model": model_name}}}} + ), + } + + @wrapper_client async def acreate_realtime_transcription_session( model: str | None = None, @@ -464,13 +482,11 @@ async def acreate_realtime_transcription_session( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - # Ensure the upstream body's input_audio_transcription.model matches the - # authorized routing model. This prevents a caller from supplying an allowed - # top-level model for auth while sneaking a different model into the nested - # transcription config that gets forwarded to the provider. - if isinstance(request_data.get("input_audio_transcription"), dict): - request_data["input_audio_transcription"]["model"] = model_name + # The upstream transcription model must match the authorized routing model, so a + # caller cannot pass auth with an allowed top-level model and forward a different one. + request_data: Final = _with_upstream_transcription_model( + req.model_dump(exclude_unset=True, exclude={"model"}), model_name + ) return await base_llm_http_handler.async_realtime_transcription_session_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 37e1b57192c..4a9b4228bc8 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Any, Literal, TypeAlias +from typing import Any, Literal from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict @@ -13,7 +13,6 @@ from .llms.openai import ( ) ALL_DELTA_TYPES = Literal["text", "audio"] -RealtimeSessionType: TypeAlias = Literal["realtime", "transcription", "translation"] class RealtimeResponseTransformInput(TypedDict): @@ -72,7 +71,7 @@ class RealtimeAudioTranscriptionConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="allow", frozen=True) model: str | None = None - delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None + delay: str | None = None keywords: Sequence[str] | None = None language: str | None = None languages: Sequence[str] | None = None @@ -114,7 +113,7 @@ class RealtimeSessionConfig(LiteLLMBaseModel): model_config = ConfigDict(extra="allow", frozen=True) - type: RealtimeSessionType | None = None + type: str | None = None model: str | None = None instructions: str | None = None audio: RealtimeSessionAudioConfig | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9bc4fad6bf0..21bffd4afac 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -44,6 +44,8 @@ from pydantic import ( JsonValue, PrivateAttr, SkipValidation, + TypeAdapter, + ValidationError, field_serializer, field_validator, ) @@ -775,10 +777,8 @@ MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset( API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), "/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), - "/openai/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), "/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), "/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), - "/openai/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), # Chat Completions "/chat/completions": [CallTypes.acompletion, CallTypes.completion], "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], @@ -1102,7 +1102,6 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/openai/v1/realtime": (CallTypes.arealtime,), "/realtime/translations": (CallTypes.arealtime,), "/v1/realtime/translations": (CallTypes.arealtime,), - "/openai/v1/realtime/translations": (CallTypes.arealtime,), "/live/sessions": (CallTypes.arealtime,), "/v1/live/sessions": (CallTypes.arealtime,), "/openai/v1/live/sessions": (CallTypes.arealtime,), @@ -2884,6 +2883,9 @@ class TranscriptionDetectedLanguage(LiteLLMBaseModel): code: str +_DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, ...]) + + class TranscriptionResponse(OpenAIObject): text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None @@ -2907,7 +2909,18 @@ class TranscriptionResponse(OpenAIObject): languages: Sequence[TranscriptionDetectedLanguage | Mapping[str, object]] | None = None, **kwargs: object, # kwargs-ok: OpenAI-compatible responses preserve provider extension fields ) -> None: - super().__init__(text=text, usage=usage, languages=languages, **kwargs) + super().__init__( + text=text, usage=usage, languages=TranscriptionResponse.detected_languages(languages), **kwargs + ) + if self.languages is None: + del self.languages + + @staticmethod + def detected_languages(value: object) -> tuple[TranscriptionDetectedLanguage, ...] | None: + try: + return _DETECTED_LANGUAGES.validate_python(value, from_attributes=True) + except ValidationError: + return None def set_audio_transcription_duration(self, duration: float) -> None: self._hidden_params["audio_transcription_duration"] = duration diff --git a/litellm/utils.py b/litellm/utils.py index 8b0d437775b..11ee0fb56c8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9781,6 +9781,7 @@ class ProviderConfigManager: def get_provider_audio_transcription_config( model: str, provider: LlmProviders, + base_model: str | None = None, ) -> BaseAudioTranscriptionConfig | None: model_cost_entry: Final = _get_model_cost_entry_for_provider_config( model=model, @@ -9808,7 +9809,7 @@ class ProviderConfigManager: return XAIAudioTranscriptionConfig() elif litellm.LlmProviders.OPENAI == provider: - if model == "gpt-transcribe": + if "gpt-transcribe" in (model, base_model): from litellm.llms.openai.transcriptions.gpt_transformation import ( OpenAIGPTTranscribeAudioTranscriptionConfig, ) diff --git a/scripts/check_openapi_docs.py b/scripts/check_openapi_docs.py index 8dd83d12912..e29cac29212 100755 --- a/scripts/check_openapi_docs.py +++ b/scripts/check_openapi_docs.py @@ -18,6 +18,7 @@ from collections.abc import Iterator, Mapping from typing import Final, NamedTuple GATED_METHODS: Final = frozenset({"post"}) +SCHEMA_MEDIA_TYPES: Final = ("application/json", "application/sdp") class Violation(NamedTuple): @@ -32,7 +33,10 @@ def _as_mapping(node: object) -> Mapping[str, object]: def _content_schema(container: object) -> object: content: Final = _as_mapping(_as_mapping(container).get("content")) - return next((schema for media in content.values() if (schema := _as_mapping(media).get("schema"))), None) + return next( + (schema for media_type in SCHEMA_MEDIA_TYPES if (schema := _as_mapping(content.get(media_type)).get("schema"))), + None, + ) def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]: diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 4c91b0b1707..320b35ff292 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3,7 +3,6 @@ import base64 import json from collections.abc import Coroutine, Mapping, Sequence from dataclasses import dataclass -from datetime import UTC, datetime from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -20,7 +19,6 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.realtime_streaming import ( RealTimeStreaming, client_sent_openai_beta_realtime_header, @@ -31,14 +29,10 @@ from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.litellm_pre_call_utils import _apply_resolved_guardrails_to_metadata -from litellm.realtime_api.main import _build_litellm_metadata from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeResponseTypedDict from litellm.types.utils import GenericGuardrailAPIInputs -from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyMatchContext -from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep def _mock_logging() -> MagicMock: @@ -517,6 +511,74 @@ async def test_backend_to_client_beta_receives_normalized_events(): assert sent["output_index"] == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("event_type", ("session.created", "session.updated")) +async def test_beta_client_bridged_to_a_ga_backend_receives_the_beta_session_shape(event_type: str) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + json.dumps( + { + "type": event_type, + "event_id": "e1", + "session": { + "type": "realtime", + "object": "realtime.session", + "id": "sess_1", + "model": "gpt-realtime-2.1-mini", + "instructions": "Answer briefly", + "output_modalities": ["audio"], + "max_output_tokens": "inf", + "tools": [], + "audio": { + "input": { + "format": {"type": "audio/pcmu"}, + "transcription": {"model": "gpt-transcribe"}, + "noise_reduction": None, + "turn_detection": {"type": "server_vad", "create_response": True}, + }, + "output": {"format": {"type": "audio/pcm", "rate": 24000}, "voice": "marin", "speed": 1.1}, + }, + }, + } + ).encode(), + ConnectionClosed(None, None), + ] + ) + logging_obj: Final = _mock_logging() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False) + + await streaming.backend_to_client_send_messages() + + sent: Final = json.loads(client_ws.send_text.call_args_list[0].args[0]) + assert sent["type"] == event_type + assert sent["session"] == { + "object": "realtime.session", + "id": "sess_1", + "model": "gpt-realtime-2.1-mini", + "instructions": "Answer briefly", + "modalities": ["audio", "text"], + "max_response_output_tokens": "inf", + "tools": [], + "input_audio_format": "g711_ulaw", + "input_audio_transcription": {"model": "gpt-transcribe"}, + "input_audio_noise_reduction": None, + "turn_detection": {"type": "server_vad", "create_response": True}, + "output_audio_format": "pcm16", + "voice": "marin", + "speed": 1.1, + }, ( + "Beta and GA session fields, checked 2026-10-20: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/beta/realtime/session.py and " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_session_create_response.py" + ) + + @pytest.mark.asyncio async def test_backend_to_client_stores_normalized_events_for_logging(): client_ws = MagicMock() @@ -1246,58 +1308,112 @@ async def test_translation_session_update_rejects_disallowed_nested_transcriptio assert client_ws.close.await_args.kwargs["code"] == 1008 +def _translation_transcription_update(transcription: Mapping[str, object]) -> str: + return json.dumps( + { + "type": "session.update", + "session": {"type": "translation", "audio": {"input": {"transcription": transcription}}}, + } + ) + + @pytest.mark.asyncio -async def test_translation_session_update_binds_nested_transcription_model() -> None: - backend_ws = MagicMock() - backend_ws.send = AsyncMock() - streaming = RealTimeStreaming( +async def test_translation_session_update_authorizes_each_new_nested_transcription_model() -> None: + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( MagicMock(), backend_ws, MagicMock(), model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth( + models=["gpt-realtime-translate", "gpt-realtime-whisper", "gpt-live-transcribe"] + ), + translation_session=True, + ) + + for transcription in ( + {"model": "gpt-realtime-whisper", "language": "en"}, + {"model": "gpt-live-transcribe", "language": "fr"}, + {"language": "de"}, + ): + await streaming._send_to_backend(_translation_transcription_update(transcription)) + + assert [ + json.loads(call.args[0])["session"]["audio"]["input"]["transcription"] + for call in backend_ws.send.await_args_list + ] == [ + {"model": "gpt-realtime-whisper", "language": "en"}, + {"model": "gpt-live-transcribe", "language": "fr"}, + {"model": "gpt-live-transcribe", "language": "de"}, + ] + + +@pytest.mark.asyncio +async def test_translation_session_update_refuses_a_later_nested_transcription_model_the_key_cannot_call() -> None: + client_ws: Final = MagicMock(send_text=AsyncMock(), close=AsyncMock()) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + model="gpt-realtime-translate", user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), translation_session=True, ) - await streaming._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": { - "type": "translation", - "audio": { - "input": { - "transcription": {"model": "gpt-realtime-whisper", "language": "en"}, - } - }, - }, - } - ) + first_sent: Final = await streaming._send_to_backend( + _translation_transcription_update({"model": "gpt-realtime-whisper", "language": "en"}) ) - await streaming._send_to_backend( - json.dumps( - { - "type": "session.update", - "session": { - "type": "translation", - "audio": { - "input": { - "transcription": {"model": "gpt-live-transcribe", "language": "fr"}, - } - }, - }, - } - ) + second_sent: Final = await streaming._send_to_backend( + _translation_transcription_update({"model": "gpt-live-transcribe", "language": "fr"}) ) - first_sent = json.loads(backend_ws.send.await_args_list[0].args[0]) - second_sent = json.loads(backend_ws.send.await_args_list[1].args[0]) - assert first_sent["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" - assert second_sent["session"]["audio"]["input"]["transcription"] == { - "model": "gpt-realtime-whisper", - "language": "fr", - } - assert streaming._is_transcription_session is False + assert (first_sent, second_sent) == (True, False) + assert backend_ws.send.await_count == 1 + error: Final = json.loads(client_ws.send_text.await_args.args[0]) + assert error["error"]["type"] == "permission_error" + assert "gpt-live-transcribe" in error["error"]["message"] + assert client_ws.close.await_args.kwargs["code"] == 1008 + + +@pytest.mark.asyncio +async def test_translation_session_update_sends_the_deployment_model_for_a_nested_transcription_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + model_list: Final = [ + { + "model_name": "whisper-alias", + "litellm_params": { + "model": "azure/qa-whisper-deployment", + "api_base": "https://realtime-qa.openai.azure.com", + "api_key": "azure-test-key", + }, + } + ] + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "whisper-alias"]), + translation_session=True, + ) + + await streaming._send_to_backend(_translation_transcription_update({"model": "whisper-alias", "language": "en"})) + await streaming._send_to_backend(_translation_transcription_update({"language": "fr"})) + + assert [ + json.loads(call.args[0])["session"]["audio"]["input"]["transcription"] + for call in backend_ws.send.await_args_list + ] == [ + {"model": "qa-whisper-deployment", "language": "en"}, + {"model": "qa-whisper-deployment", "language": "fr"}, + ] @pytest.mark.asyncio @@ -3279,80 +3395,7 @@ async def test_translation_rejects_applicable_guardrails_before_forwarding_audio await streaming.bidirectional_forward() messages: Final = tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) - if default_on or requested: - assert messages[0]["error"]["type"] == "guardrail_error" - client.close.assert_awaited_once_with( - code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" - ) - backend.send.assert_not_awaited() - else: - assert output_event in messages - backend.send.assert_awaited_once_with(input_event) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("metadata_bucket", ("metadata", "litellm_metadata")) -@pytest.mark.parametrize("mode", ("pre_call", "post_call")) -@pytest.mark.parametrize("selected,condition_matches", ((True, True), (True, False), (False, True))) -async def test_translation_rejects_selected_pipeline_guardrails_before_forwarding_audio( - monkeypatch: pytest.MonkeyPatch, - metadata_bucket: str, - mode: Literal["pre_call", "post_call"], - selected: bool, - condition_matches: bool, -) -> None: - guardrail: Final = CustomGuardrail( - guardrail_name="speech-policy", event_hook=GuardrailEventHooks(mode), default_on=False - ) - monkeypatch.setattr(litellm, "callbacks", [guardrail]) - request_data: Final[dict[str, object]] = {"metadata": {}, "litellm_metadata": {"user_id": "qa-user"}} - _apply_resolved_guardrails_to_metadata( - data=request_data, - metadata_variable_name=metadata_bucket, - context=PolicyMatchContext(model="qa-translation"), - policy_names=["translation-policy"] if selected else [], - policies={ - "translation-policy": Policy( - condition=PolicyCondition(model="qa-translation" if condition_matches else "unrelated-model"), - pipeline=GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="speech-policy", on_fail="block")]), - ) - }, - ) - input_event: Final = json.dumps( - {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(4800)).decode()} - ) - output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(4800)).decode()} - client: Final = _ga_client_ws() - client.receive_text = AsyncMock(side_effect=[input_event, ConnectionClosed(None, None)]) - client.send_text = AsyncMock() - client.close = AsyncMock() - backend: Final = MagicMock() - backend.send = AsyncMock() - backend.recv = AsyncMock(side_effect=[json.dumps(output_event), ConnectionClosed(None, None)]) - logging_obj: Final = Logging( - model="gpt-realtime-translate", - messages=[], - stream=True, - call_type="_arealtime", - start_time=datetime(2026, 1, 1, tzinfo=UTC), - litellm_call_id="pipeline-rejection", - function_id="pipeline-rejection", - ) - worker: Final = _InlineLoggingWorker() - streaming: Final = RealTimeStreaming( - websocket=client, - backend_ws=backend, - logging_obj=logging_obj, - translation_session=True, - request_data={"litellm_metadata": _build_litellm_metadata(request_data)}, - logging_worker=worker, - ) - - await streaming.bidirectional_forward() - await worker.drain() - - messages: Final = tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) - if selected and condition_matches: + if (default_on or requested) and event_hook is not GuardrailEventHooks.post_call: assert messages[0]["error"]["type"] == "guardrail_error" client.close.assert_awaited_once_with( code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" diff --git a/tests/unit/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py index 675b7f8c1f1..f86fae494ef 100644 --- a/tests/unit/llms/azure/test_audio_transcriptions.py +++ b/tests/unit/llms/azure/test_audio_transcriptions.py @@ -163,7 +163,39 @@ async def test_azure_transcription_parses_response_formats( assert response.text == expected_text -@pytest.mark.parametrize("model", ["whisper-1", "gpt-transcribe", "custom-transcribe-deployment"]) +@pytest.mark.parametrize("api_version", ["2024-06-01", "2025-03-01-preview", "v1", "latest", "preview"]) +async def test_azure_router_transcription_sends_the_default_api_version( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + api_version: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("AZURE_API_VERSION", raising=False) + route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "transcriber", + "litellm_params": { + "model": "azure/gpt-transcribe", + "api_key": "test-key", + "api_base": "https://transcribe-route.openai.azure.com", + "api_version": api_version, + }, + } + ] + ) + + response: Final = await router.atranscription(model="transcriber", file=_audio_file()) + + sent_url: Final = route.calls.last.request.url + assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions" + assert sent_url.params.get("api-version") == litellm.AZURE_DEFAULT_API_VERSION + assert response.text == "hello" + + @pytest.mark.parametrize( ("api_version", "expected_api_version"), [ @@ -173,20 +205,19 @@ async def test_azure_transcription_parses_response_formats( ("2025-03-01-preview", "2025-03-01-preview"), ], ) -async def test_azure_transcription_uses_the_deployments_route( +def test_azure_sync_transcription_uses_the_deployments_route( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, - model: str, api_version: str, expected_api_version: str, ) -> None: - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("AZURE_API_VERSION", raising=False) route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock( return_value=httpx.Response(200, json={"text": "hello"}) ) - response: Final = await litellm.atranscription( - model=f"azure/{model}", + response: Final = litellm.transcription( + model="azure/gpt-transcribe", file=_audio_file(), api_key="test-key", api_base="https://transcribe-route.openai.azure.com", @@ -194,6 +225,6 @@ async def test_azure_transcription_uses_the_deployments_route( ) sent_url: Final = route.calls.last.request.url - assert sent_url.path == f"/openai/deployments/{model}/audio/transcriptions" + assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions" assert sent_url.params.get("api-version") == expected_api_version assert response.text == "hello" diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index df5f96d6581..4963c5f9a20 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -13,6 +13,7 @@ from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, Open import litellm from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.openai.transcriptions.gpt_transformation import ( OpenAIGPTTranscribeAudioTranscriptionConfig, ) @@ -64,7 +65,7 @@ def test_gpt_transcribe_optional_params_are_preserved(): def test_transcription_response_preserves_empty_languages(): response = TranscriptionResponse(text="hello", languages=[]) - assert response.model_dump()["languages"] == [] + assert json.loads(response.model_dump_json())["languages"] == [] def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: @@ -97,7 +98,7 @@ def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: logged_response: Final = logging_obj.success_handler.call_args.args[0] assert logged_response.text == "hello world" assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5} - assert logged_response.model_dump()["languages"] == [{"code": "en"}] + assert json.loads(logged_response.model_dump_json())["languages"] == [{"code": "en"}] logging_obj.failure_handler.assert_not_called() @@ -162,38 +163,7 @@ async def test_openai_handler_returns_native_typed_stream(): logging_obj.async_success_handler.assert_awaited_once() logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] assert logged_response.text == "hello world" - assert logged_response.languages == [] - - -@pytest.mark.asyncio -async def test_closed_transcription_stream_without_usage_or_duration_does_not_log_success(): - async def send_response(request: httpx.Request) -> httpx.Response: - events = ( - {"type": "transcript.text.delta", "delta": "hello"}, - {"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 1}}, - ) - content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() - return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) - - http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) - client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) - stream = await client.audio.transcriptions.create( - model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True - ) - logging_obj = MagicMock() - logging_obj.async_success_handler = AsyncMock() - logging_obj.async_failure_handler = AsyncMock() - wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now()) - - async for event in wrapped_stream: - assert event.type == "transcript.text.delta" - break - await wrapped_stream.close() - await client.close() - - logging_obj.handle_sync_success_callbacks_for_async_calls.assert_not_called() - logging_obj.async_success_handler.assert_not_awaited() - logging_obj.async_failure_handler.assert_not_awaited() + assert logged_response.languages == () @pytest.mark.asyncio @@ -249,19 +219,6 @@ async def test_atranscription_stream_preserves_duration_for_callback_cost(): assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) -def test_gpt_transcribe_rejects_conflicting_language_inputs(): - audio_file = io.BytesIO(b"audio") - audio_file.name = "sample.wav" - with pytest.raises(litellm.UnsupportedParamsError, match="cannot be used together"): - litellm.transcription( - model="gpt-transcribe", - file=audio_file, - language="en", - languages=["fr"], - api_key="sk-test", - ) - - @pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"]) def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None: audio_file = io.BytesIO(b"audio") @@ -350,16 +307,49 @@ def test_gpt_transcribe_sends_text_response_format_to_the_provider( assert b'name="response_format"\r\n\r\ntext\r\n' in sent_bodies[0] -def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: None) -> None: - audio_file = io.BytesIO(b"audio") - audio_file.name = "sample.wav" - with pytest.raises(litellm.UnsupportedParamsError, match="Realtime API"): - litellm.transcription( - model="gpt-live-transcribe", - file=audio_file, - api_key="sk-test", +def test_file_transcription_reaches_models_whose_cost_map_lists_other_endpoints( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + sent_requests: Final[list[httpx.Request]] = [] # mutable-ok: collects what the mock upstreams received + + def send_gemini_response(request: httpx.Request) -> httpx.Response: + sent_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "interaction-1", + "status": "completed", + "steps": [ + {"type": "model_generation", "content": [{"type": "text", "text": "hello", "annotations": []}]} + ], + }, ) + def send_whisper_response(request: httpx.Request) -> httpx.Response: + sent_requests.append(request) + return httpx.Response(200, json={"text": "hello"}) + + gemini_response: Final = litellm.transcription( + model="gemini/gemini-2.5-flash", + file=_one_second_wav(), + api_key="gemini-test-key", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send_gemini_response))), + ) + monkeypatch.setattr(litellm, "client_session", httpx.Client(transport=httpx.MockTransport(send_whisper_response))) + whisper_response: Final = litellm.transcription( + model="openai/gpt-realtime-whisper", + file=_one_second_wav(), + api_key="self-hosted-whisper-key", + api_base="https://whisper.self-hosted.example/v1", + ) + + assert gemini_response.text == "hello" + assert whisper_response.text == "hello" + assert tuple(str(request.url.host) for request in sent_requests) == ( + "generativelanguage.googleapis.com", + "whisper.self-hosted.example", + ) + def test_azure_gpt_transcribe_uses_deployment_scoped_route(): def send_response(request: httpx.Request) -> httpx.Response: @@ -616,3 +606,160 @@ def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_ca assert unstreamed_spend.cost > 0 assert unstreamed_spend.usage assert streamed_spend == unstreamed_spend + + +class _LoggedResponseRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.responses: Final[queue.Queue[object]] = queue.Queue() + + def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.responses.put(kwargs["standard_logging_object"]["response"]) + + +@pytest.mark.parametrize( + "provider_body", + ({"text": "hello"}, {"text": "hello", "languages": ["en"]}, {"text": "hello", "languages": None}), +) +def test_transcription_without_parsed_languages_has_no_languages_key( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object] +) -> None: + recorder: Final = _LoggedResponseRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with _openai_transcription_client( + httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body)) + ) as client: + response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + logged_response: Final = recorder.responses.get(timeout=5) + + assert response.text == "hello" + assert "languages" not in json.loads(response.model_dump_json()) + assert isinstance(logged_response, dict) + assert "languages" not in logged_response + + +def test_unstreamed_transcription_sends_no_stream_field(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello"}) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="stream"' not in sent_bodies[0] + + +def test_transcription_forwards_both_language_and_languages(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}}) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), language="en", languages=["fr"], client=client + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="language"\r\n\r\nen\r\n' in sent_bodies[0] + assert b'name="languages[]"\r\n\r\nfr\r\n' in sent_bodies[0] + + +def _gpt_transcribe_cost_per_second() -> float: + rate: Final = litellm.model_cost["gpt-transcribe"]["input_cost_per_second"] + assert isinstance(rate, float) + return rate + + +async def _spend_for_one_second_wav( + monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object], router: litellm.Router | None +) -> _LoggedSpend: + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body)) + ), + ) + if router is None: + await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + else: + await router.atranscription(model="my-transcribe", file=_one_second_wav(), client=client) + spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await client.close() + return spend + + +@pytest.mark.asyncio +async def test_custom_named_gpt_transcribe_deployment_bills_the_provider_seconds( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "my-transcribe", + "litellm_params": {"model": "openai/my-transcribe", "api_key": "sk-test"}, + "model_info": {"base_model": "gpt-transcribe"}, + } + ] + ) + + spend: Final = await _spend_for_one_second_wav( + monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 2}}, router + ) + + assert spend.cost == pytest.approx(2 * _gpt_transcribe_cost_per_second()) + + +@pytest.mark.asyncio +async def test_gpt_transcribe_bills_the_file_length_when_the_provider_reports_zero_seconds( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + spend: Final = await _spend_for_one_second_wav( + monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 0}}, None + ) + + assert spend.cost == pytest.approx(_gpt_transcribe_cost_per_second()) + + +@pytest.mark.asyncio +async def test_m4a_transcription_stream_that_ends_early_logs_one_spend_row_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + if not _asks_for_stream(await request.aread()): + return httpx.Response(200, json={"text": "hello"}) + return httpx.Response( + 200, + content=f"data: {json.dumps({'type': 'transcript.text.delta', 'delta': 'hello'})}\n\n".encode(), + headers={"content-type": "text/event-stream"}, + ) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), client=client) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), stream=True, client=client + ) + events: Final = [event.type async for event in stream] + streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await asyncio.sleep(0.2) + await client.close() + + assert events == ["transcript.text.delta"] + assert streamed_spend.cost == unstreamed_spend.cost + assert recorder.async_spends.empty() diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index 9a75f1f442d..3d443c251a3 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -476,9 +476,7 @@ def test_managed_live_upgrade_defers_the_model_to_the_session_start_frame(route: ) == {"model": "frame-model"} -@pytest.mark.parametrize( - "route", ("/realtime/translations", "/v1/realtime/translations", "/openai/v1/realtime/translations") -) +@pytest.mark.parametrize("route", ("/realtime/translations", "/v1/realtime/translations")) @pytest.mark.parametrize("query_model", (None, "requested")) def test_managed_translation_authorizes_the_same_model_as_the_endpoint(route: str, query_model: str | None) -> None: from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 622350579a1..36fb98e4a4f 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -788,13 +788,10 @@ def test_realtime_webrtc_http_routes_classified_as_llm_api(route): [ "/realtime/translations", "/v1/realtime/translations", - "/openai/v1/realtime/translations", "/realtime/translations/client_secrets", "/v1/realtime/translations/client_secrets", - "/openai/v1/realtime/translations/client_secrets", "/realtime/translations/calls", "/v1/realtime/translations/calls", - "/openai/v1/realtime/translations/calls", ], ) def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route): @@ -814,6 +811,24 @@ def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route): ) +@pytest.mark.parametrize( + "route", + [ + "/openai/v1/realtime/translations", + "/openai/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/calls", + ], +) +def test_openai_prefixed_realtime_translation_routes_denied_for_openai_virtual_keys(route: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes"]) + + with pytest.raises(HTTPException) as exc_info: + RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) + + assert exc_info.value.status_code == 403 + assert f"Tried to call route: {route}" in str(exc_info.value.detail) + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index df2c6aa71af..373636f7a8c 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -11937,9 +11937,7 @@ async def test_websocket_auth_hands_the_reservation_to_the_socket_state(): @pytest.mark.asyncio -@pytest.mark.parametrize( - "path", ("/realtime/translations", "/v1/realtime/translations", "/openai/v1/realtime/translations") -) +@pytest.mark.parametrize("path", ("/realtime/translations", "/v1/realtime/translations")) @pytest.mark.parametrize("query", (b"", b"model=gpt-realtime-translate"), ids=("default_model", "explicit_model")) async def test_translation_websocket_auth_enforces_the_default_model_budget( monkeypatch: pytest.MonkeyPatch, path: str, query: bytes @@ -11989,6 +11987,52 @@ async def test_translation_websocket_auth_enforces_the_default_model_budget( assert "budget" in str(denied.value.detail).lower() +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime/translations/client_secrets", "/v1/realtime/translations/client_secrets")) +async def test_translation_client_secret_auth_enforces_the_session_model_budget( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + model: Final = "gpt-realtime-translate" + token: Final = UserAPIKeyAuth( + token="test-key", + models=[model], + model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0}) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + body: Final = json.dumps({"session": {"type": "translation", "model": model}}).encode() + request: Final = Request( + scope={ + "type": "http", + "method": "POST", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test"), (b"content-type", b"application/json")], + "query_string": b"", + }, + receive=AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False}), + ) + + with pytest.raises(ProxyException) as denied: + await user_api_key_auth(request=request, api_key="Bearer sk-test") + + assert denied.value.type == "budget_exceeded" + assert model in denied.value.message + + @pytest.mark.asyncio @pytest.mark.parametrize("path", ("/realtime", "/v1/realtime", "/openai/v1/realtime")) async def test_managed_agent_transcription_websocket_without_a_model_is_refused( diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index cd5820fa2fd..e78a1dce157 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -685,6 +685,17 @@ async def test_get_form_data(): assert result["temperature"] == "0.2" assert result["include"] == ["logprobs"] assert result["timestamp_granularities"] == ["word", "segment"] + assert tuple(result) == ( + "file", + "model", + "include", + "language", + "prompt", + "response_format", + "stream", + "temperature", + "timestamp_granularities", + ) def test_get_tags_from_request_body_with_metadata_tags(): diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index d16f5dd8a5d..dd9e87c7964 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -10,7 +10,8 @@ Pins (PR2): from __future__ import annotations import io -from collections.abc import Callable, Mapping +import json +from collections.abc import AsyncIterator, Callable, Mapping from contextlib import AbstractContextManager from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -475,3 +476,114 @@ def test_streamed_transcription_honors_a_deployment_post_call_guardrail( else: assert response.headers["content-type"].startswith("text/event-stream") assert '"text":"my pin is 1234"' in response.text + + +def _streams_chat_and_transcription(request: httpx.Request) -> httpx.Response: + sse_body: Final = ( + b'data: {"id":"chat-1","object":"chat.completion.chunk","created":0,"model":"gpt-4o",' + b'"choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n' + if request.url.path.endswith("/chat/completions") + else b'data: {"type":"transcript.text.done","text":"hello world"}\n\n' + ) + return httpx.Response( + 200, + content=sse_body, + headers={"content-type": "text/event-stream", "x-ratelimit-remaining-requests": "99"}, + ) + + +def _litellm_header_names(response: httpx.Response) -> frozenset[str]: + return frozenset(name for name in response.headers if name.startswith("x-litellm-")) + + +def test_streamed_transcription_carries_the_headers_a_streamed_chat_completion_carries( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.route(host="stream-upstream.example").mock(side_effect=_streams_chat_and_transcription) + upstream: Final = {"api_key": "sk-test", "api_base": "https://stream-upstream.example/v1"} + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", **upstream}}, + {"model_name": "gpt-transcribe", "litellm_params": {"model": "openai/gpt-transcribe", **upstream}}, + ] + ), + ) + + with auth_as(): + chat: Final = client.post( + "/v1/chat/completions", + json={"model": "gpt-4o", "stream": True, "messages": [{"role": "user", "content": "hi"}]}, + ) + transcription: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert chat.status_code == 200 + assert transcription.status_code == 200 + assert transcription.headers["content-type"].startswith("text/event-stream") + assert _litellm_header_names(chat) - _litellm_header_names(transcription) <= {"x-litellm-overhead-duration-ms"} + assert transcription.headers["x-litellm-model-name"] == "openai/gpt-transcribe" + assert transcription.headers["llm_provider-x-ratelimit-remaining-requests"] == "99" + assert ("x-litellm-response-cost" in transcription.headers) == ("x-litellm-response-cost" in chat.headers) + + +class _FailsAfterTwoDeltas(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + yield b'data: {"type":"transcript.text.delta","delta":"world"}\n\n' + raise httpx.ReadError("upstream connection dropped") + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_error_mid_stream_ends_with_an_error_event_and_runs_the_failure_hook( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, stream=_FailsAfterTwoDeltas(), headers={"content-type": "text/event-stream"} + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription(model="openai/gpt-transcribe", file=data["file"], stream=True, client=upstream) + + failure_hook: Final = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "post_call_failure_hook", failure_hook) + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + frames: Final = tuple( + json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: ") + ) + assert response.status_code == 200 + assert tuple(frame.get("delta") for frame in frames[:2]) == ("hello ", "world") + assert len(frames) == 3 + assert "upstream connection dropped" in frames[2]["error"]["message"] + assert set(frames[2]["error"]) == {"message", "type", "param", "code"} + failure_hook.assert_awaited_once() + assert isinstance(failure_hook.await_args.kwargs["original_exception"], httpx.ReadError) diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 645d94fe142..8eada379418 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -7,7 +7,7 @@ Tests for LiteLLM proxy realtime WebRTC HTTP endpoints: import functools import json import time -from collections.abc import Awaitable +from collections.abc import Awaitable, Mapping from typing import Final, Protocol from unittest.mock import AsyncMock, MagicMock, patch @@ -863,20 +863,6 @@ async def test_realtime_transcription_websocket_default_model_checks_key_scope() assert "is not available for this API key" in close_kwargs["reason"] -def test_realtime_transcription_upstream_query_omits_model(): - from litellm.proxy import proxy_server - - assert ( - proxy_server._resolve_realtime_upstream_query_model( - model="gpt-live-transcribe", - intent="transcription", - is_translation=False, - route_model="gpt-live-transcribe", - ) - is None - ) - - @pytest.mark.asyncio async def test_realtime_transcription_websocket_default_model_checks_team_scope(): from litellm.proxy import proxy_server @@ -1619,6 +1605,61 @@ _AZURE_REALTIME_DEPLOYMENTS: Final = ( ) +def _post_through_the_azure_deployments( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: object, +) -> tuple[httpx.Response, respx.Route]: + from litellm.proxy import proxy_server + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_list: Final = [ # mutable-ok: Router takes a list of deployments + { + "model_name": alias, + "litellm_params": { + "model": deployment, + "api_base": "https://realtime-qa.openai.azure.com", + "api_key": "azure-test-key", + }, + } + for alias, deployment in _AZURE_REALTIME_DEPLOYMENTS + ] + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + upstream: Final = respx_mock.route(host="realtime-qa.openai.azure.com").mock( + return_value=httpx.Response(200, json={"value": "ek_azure", "expires_at": int(time.time()) + 3600}) + ) + monkeypatch.setitem( + proxy_app.dependency_overrides, + user_api_key_auth, + lambda: UserAPIKeyAuth(user_id="test-user", models=[alias for alias, _ in _AZURE_REALTIME_DEPLOYMENTS]), + ) + with ( + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response: Final = TestClient(proxy_app).post( + path, headers={"Authorization": "Bearer sk-test-master-key"}, json=body + ) + return response, upstream + + +def _upstream_session_value(upstream: respx.Route, session_path: tuple[str, ...]) -> object: + return functools.reduce( + lambda node, key: node[key], session_path, json.loads(upstream.calls.last.request.content)["session"] + ) + + @pytest.mark.parametrize( ("path", "body", "upstream_model_path", "expected_model"), ( @@ -1651,6 +1692,15 @@ _AZURE_REALTIME_DEPLOYMENTS: Final = ( ("model",), "qa-translate-deployment", ), + ( + "/v1/realtime/translations/client_secrets", + { + "model": "azure-translate-alias", + "session": {"audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}}}, + }, + ("audio", "input", "transcription", "model"), + "qa-whisper-deployment", + ), ), ) def test_client_secrets_mint_a_secret_for_the_azure_deployment_model( @@ -1664,46 +1714,95 @@ def test_client_secrets_mint_a_secret_for_the_azure_deployment_model( upstream_model_path: tuple[str, ...], expected_model: str, ) -> None: - from litellm.proxy import proxy_server - - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - model_list: Final = [ # mutable-ok: Router takes a list of deployments - { - "model_name": alias, - "litellm_params": { - "model": deployment, - "api_base": "https://realtime-qa.openai.azure.com", - "api_key": "azure-test-key", - }, - } - for alias, deployment in _AZURE_REALTIME_DEPLOYMENTS - ] - monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) - monkeypatch.setattr(proxy_server, "llm_model_list", model_list) - upstream: Final = respx_mock.route(host="realtime-qa.openai.azure.com").mock( - return_value=httpx.Response(200, json={"value": "ek_azure", "expires_at": int(time.time()) + 3600}) + response, upstream = _post_through_the_azure_deployments( + proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body ) - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", models=[alias for alias, _ in _AZURE_REALTIME_DEPLOYMENTS] - ) - try: - with ( - patch( # test-quality-ok: endpoint test isolates request metadata enrichment - "litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data - ), - patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger - "litellm.proxy.proxy_server.proxy_logging_obj" - ) as logging, - ): - logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) - logging.post_call_failure_hook = AsyncMock() - response: Final = TestClient(proxy_app).post( - path, headers={"Authorization": "Bearer sk-test-master-key"}, json=body - ) - finally: - proxy_app.dependency_overrides.pop(user_api_key_auth, None) assert response.status_code == 200, response.text - upstream_session: Final = json.loads(upstream.calls.last.request.content)["session"] - upstream_model: Final = functools.reduce(lambda node, key: node[key], upstream_model_path, upstream_session) - assert upstream_model == expected_model + assert _upstream_session_value(upstream, upstream_model_path) == expected_model + + +@pytest.mark.parametrize( + ("body", "session_path", "expected"), + ( + ( + { + "session": { + "type": "realtime", + "model": "azure-realtime-alias", + "audio": {"input": {"turn_detection": None}}, + } + }, + ("audio", "input", "turn_detection"), + None, + ), + ( + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}, "turn_detection": None}}, + } + }, + ("audio", "input", "turn_detection"), + None, + ), + ({"session": {"type": "conversation", "model": "azure-realtime-alias"}}, ("type",), "realtime"), + ( + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias", "delay": "auto"}}}, + } + }, + ("audio", "input", "transcription", "delay"), + "auto", + ), + ), +) +def test_client_secrets_forward_every_session_value_the_caller_set( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + body: Mapping[str, object], + session_path: tuple[str, ...], + expected: object, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + body, + ) + + assert response.status_code == 200, response.text + assert _upstream_session_value(upstream, session_path) == expected + + +@pytest.mark.parametrize( + ("path", "body"), + ( + ("/v1/realtime/client_secrets", {"session": "not-a-session"}), + ("/v1/realtime/translations/client_secrets", {"session": {"audio": {"input": "not-an-input"}}}), + ("/v1/realtime/transcription_sessions", {"audio": {"input": {"transcription": "not-a-transcription"}}}), + ), +) +def test_realtime_session_routes_answer_400_for_a_malformed_body( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: Mapping[str, object], +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body + ) + + assert (response.status_code, response.json()["error"]["type"]) == (400, "invalid_request_error") + assert upstream.called is False diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index f8ea1e5d075..fabbf57b135 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -39,6 +39,7 @@ from fastapi import FastAPI, HTTPException, WebSocket # test /chat/completion request to the proxy from fastapi.testclient import TestClient +from starlette.testclient import WebSocketTestSession from litellm.integrations.custom_logger import CustomLogger from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -3459,6 +3460,91 @@ def test_live_session_pre_call_abort_releases_reservations( route_call.assert_not_awaited() +def _receive_until_closed(websocket: WebSocketTestSession) -> None: + while True: + websocket.receive_text() + + +@pytest.mark.parametrize( + ("litellm_params", "expected_upstream_url"), + ( + pytest.param( + {"model": "openai/gateway-transcribe", "api_base": "https://gateway.example/v1"}, + "wss://gateway.example/v1/realtime?model=gateway-transcribe&intent=transcription", + id="gateway", + ), + pytest.param( + {"model": "xai/grok-voice", "api_base": "https://api.x.ai/v1"}, + "wss://api.x.ai/v1/realtime?model=grok-voice&intent=transcription", + id="xai", + ), + pytest.param( + {"model": "openai/gpt-live-transcribe", "api_base": "https://api.openai.com/v1"}, + "wss://api.openai.com/v1/realtime?intent=transcription", + id="openai", + ), + ), +) +def test_realtime_transcription_websocket_keeps_the_model_upstream_except_on_openai_hosts( + client_no_auth: TestClient, + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, str], + expected_upstream_url: str, +) -> None: + from starlette.websockets import WebSocketDisconnect + + from litellm.proxy import proxy_server + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[{"model_name": "team-transcriber", "litellm_params": {**litellm_params, "api_key": "fake"}}] + ), + ) + connect: Final = MagicMock(side_effect=RuntimeError("upstream url captured")) + monkeypatch.setattr("websockets.connect", connect) + + with ( + client_no_auth.websocket_connect( + "/v1/realtime?model=team-transcriber&intent=transcription", + headers={"Authorization": "Bearer test-api-key"}, + ) as websocket, + pytest.raises(WebSocketDisconnect), + ): + _receive_until_closed(websocket) + + assert connect.call_args.args[0] == expected_upstream_url + + +def test_openai_prefixed_translation_websocket_takes_the_openai_websocket_pass_through( + client_no_auth: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.websockets import WebSocketDisconnect + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _OPENAI_WS_DISABLED_REFUSAL + + route_call: Final = AsyncMock() + monkeypatch.setattr(proxy_server, "route_request", route_call) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setitem(proxy_server.app.dependency_overrides, user_api_key_auth_websocket, lambda: UserAPIKeyAuth()) + + with client_no_auth.websocket_connect( + "/openai/v1/realtime/translations?model=gpt-realtime-translate", + headers={"Authorization": "Bearer test-api-key"}, + ) as websocket: + refusal: Final = websocket.receive_json() + with pytest.raises(WebSocketDisconnect) as closed: + websocket.receive_json() + + assert refusal["error"]["message"] == _OPENAI_WS_DISABLED_REFUSAL.message + assert closed.value.code == 1008 + route_call.assert_not_awaited() + + @pytest.mark.parametrize("path", ["/openai/v1/live/sessions", "/v1/live/sessions", "/live/sessions"]) def test_live_session_invalid_start_is_rejected_without_routing( path: str, diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index fc5a8f9dab4..6730ea0cf99 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10772,7 +10772,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): def test_realtime_websocket_route_aliases_registered(): - """Realtime sessions reach the proxy via six path aliases stacked on + """Realtime sessions reach the proxy via five path aliases stacked on `realtime_websocket_endpoint`. Dropping any of them silently 405s WebSocket upgrades because the catch-all `/openai/{endpoint:path}` HTTP passthrough only declares HTTP methods. The aliases must also be @@ -10792,7 +10792,6 @@ def test_realtime_websocket_route_aliases_registered(): "/openai/v1/realtime", "/v1/realtime", "/realtime", - "/openai/v1/realtime/translations", "/v1/realtime/translations", "/realtime/translations", ): diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 5d5e0c285e1..a0cdadff982 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1,4 +1,5 @@ import asyncio +import functools import ipaddress import json import ssl @@ -9,7 +10,7 @@ from datetime import datetime, timedelta, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from types import TracebackType -from typing import Final +from typing import ClassVar, Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -793,6 +794,89 @@ async def test_realtime_http_routes_send_a_bare_model_name_to_its_explicit_provi assert gateway.url == f"https://realtime-gateway.example/v1/realtime/{path}" +class _GatewayThatRecordsTheBody: + body: object = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.body = json.loads(request.content) + return httpx.Response(200, json={"client_secret": {"value": "ek_test"}}) + + +@pytest.mark.asyncio +async def test_transcription_session_sends_the_deployment_model_upstream_in_every_transcription_shape() -> None: + gateway: Final = _GatewayThatRecordsTheBody() + await realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session={ + "audio": {"input": {"transcription": {"model": "team-transcriber", "language": "en"}}}, + "input_audio_transcription": {"model": "team-transcriber"}, + "input_audio_format": "pcm16", + }, + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + assert gateway.body == { + "audio": {"input": {"transcription": {"model": "gpt-transcribe", "language": "en"}}}, + "input_audio_transcription": {"model": "gpt-transcribe"}, + "input_audio_format": "pcm16", + } + + +def _create_realtime_session_with_an_explicit_null(route: str, client: AsyncHTTPHandler) -> Awaitable[httpx.Response]: + if route == "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + session={"type": "realtime", "model": "gpt-realtime", "audio": {"input": {"turn_detection": None}}}, + custom_llm_provider="openai", + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=client, + ) + if route == "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + session={"audio": {"input": {"noise_reduction": None}}}, + custom_llm_provider="openai", + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=_realtime_http_logging(route), + client=client, + ) + return realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session={"audio": {"input": {"turn_detection": None}}}, + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=client, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "body_path"), + ( + ("client_secrets", ("session", "audio", "input", "turn_detection")), + ("translation_client_secrets", ("session", "audio", "input", "noise_reduction")), + ("transcription_sessions", ("audio", "input", "turn_detection")), + ), +) +async def test_realtime_session_requests_forward_an_explicit_null_upstream( + route: str, body_path: tuple[str, ...] +) -> None: + gateway: Final = _GatewayThatRecordsTheBody() + + await _create_realtime_session_with_an_explicit_null( + route, AsyncHTTPHandler(transport=httpx.MockTransport(gateway)) + ) + + parent: Final = functools.reduce(lambda node, key: node[key], body_path[:-1], gateway.body) + assert body_path[-1] in parent + assert parent[body_path[-1]] is None + + class _ClientWebSocketWithHeaders: def __init__(self, headers: tuple[tuple[bytes, bytes], ...]) -> None: self.scope: Final = {"headers": headers} @@ -1717,6 +1801,39 @@ async def test_translation_websocket_closes_1008_for_a_guarded_request_before_th ] +class _NativeLifecycleGuardrail(CustomGuardrail): + use_native_lifecycle_hooks: ClassVar[bool] = True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("guardrail_class", "event_hook"), + ((_NativeLifecycleGuardrail, GuardrailEventHooks.pre_call), (CustomGuardrail, GuardrailEventHooks.post_call)), +) +async def test_translation_websocket_opens_under_a_default_on_guardrail_the_realtime_runner_skips( + guardrail_class: type[CustomGuardrail], event_hook: GuardrailEventHooks, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + litellm, + "callbacks", + [guardrail_class(guardrail_name="runner-skipped", event_hook=event_hook, default_on=True)], + ) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + ) + + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + + @pytest.mark.asyncio async def test_translation_websocket_closes_1008_for_a_provider_other_than_openai_and_azure() -> None: websocket: Final = _RecordingClientWebSocket() diff --git a/tests/unit/test_check_openapi_docs.py b/tests/unit/test_check_openapi_docs.py index ef8cf0e9dbd..5d8803c4118 100644 --- a/tests/unit/test_check_openapi_docs.py +++ b/tests/unit/test_check_openapi_docs.py @@ -1,5 +1,6 @@ """Tests for scripts/check_openapi_docs.py, the scan behind scripts/openapi_docs_gate.py.""" +from collections.abc import Mapping from typing import Final import pytest @@ -11,19 +12,25 @@ _RESPONSES: Final = {"200": {"content": {"application/json": {"schema": {"$ref": _DOCUMENTED: Final = {"description": "Does a thing", "requestBody": _BODY, "responses": _RESPONSES} -@pytest.mark.parametrize( - ("request_media_type", "response_media_type"), - (("application/json", "application/json"), ("application/sdp", "application/sdp"), ("multipart/form-data", "audio/wav")), -) -def test_a_fully_documented_post_has_no_findings(request_media_type: str, response_media_type: str) -> None: - operation: Final = { +def _post_with_media_types(request_media_type: str, response_media_type: str) -> Mapping[str, object]: + return { **_DOCUMENTED, "requestBody": {"content": {request_media_type: {"schema": {"type": "string"}}}}, "responses": {"200": {"content": {response_media_type: {"schema": {"type": "string"}}}}}, } + + +@pytest.mark.parametrize("media_type", ("application/json", "application/sdp")) +def test_a_fully_documented_post_has_no_findings(media_type: str) -> None: + operation: Final = _post_with_media_types(media_type, media_type) assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == () +def test_a_schema_under_another_media_type_still_counts_as_undocumented() -> None: + operation: Final = _post_with_media_types("multipart/form-data", "audio/wav") + assert {v.code for v in checker.violations({"paths": {"/v1/thing": {"post": operation}}})} == {"OA001", "OA003"} + + def test_each_missing_piece_is_its_own_rule() -> None: bare: Final = {"summary": "Thing", "responses": {"200": {"description": "Successful Response"}}} codes: Final = {v.code for v in checker.violations({"paths": {"/v1/thing": {"post": bare}}})} diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 017c5a1364c..c23f30f3382 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -9999,6 +9999,92 @@ def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) +def _translation_session_cost(deployment_id: str) -> float: + one_minute_session: Final = LiteLLMRealtimeStreamLoggingObject( + usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), + results=[ + {"type": "session.created", "session": {"model": "gpt-realtime-translate"}}, + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}}, + ], + ) + return completion_cost( + completion_response=one_minute_session, + model="gpt-realtime-translate", + call_type=CallTypes.arealtime.value, + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + ) + + +def test_realtime_translation_deployment_per_second_rate_replaces_both_public_rates(_local_model_cost_map) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/gpt-realtime-translate", + "api_key": "sk-test", + "input_cost_per_second": 0.001, + }, + "model_info": {"id": "translate-input-priced"}, + } + ] + ) + + assert _translation_session_cost("translate-input-priced") == pytest.approx(60 * 0.001) + + +def test_realtime_translation_deployment_without_per_second_rates_bills_public_rates(_local_model_cost_map) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/gpt-realtime-translate", + "api_key": "sk-test", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + "model_info": {"id": "translate-token-priced"}, + } + ] + ) + public_entry: Final = litellm.model_cost["gpt-realtime-translate"] + public_cost: Final = 60 * (public_entry.get("input_cost_per_second") or 0.0) + 60 * ( + public_entry.get("output_cost_per_second") or 0.0 + ) + + assert public_cost > 0 + assert _translation_session_cost("translate-token-priced") == pytest.approx(public_cost) + + +def test_realtime_translation_spend_usage_carries_the_billed_output_seconds(_local_model_cost_map) -> None: + logging_obj: Final = Logging( + model="gpt-realtime-translate", + messages=[], + stream=False, + call_type=CallTypes.arealtime.value, + start_time=datetime.datetime.now(), + litellm_call_id="translation-spend-usage", + function_id="translation-spend-usage", + ) + logging_obj.update_environment_variables( + model="gpt-realtime-translate", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 30.0, "output_seconds": 42.5}} + ] + + logging_obj.success_handler(result=events, start_time=datetime.datetime.now(), end_time=datetime.datetime.now()) + + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert payload["metadata"]["usage_object"]["seconds"] == pytest.approx(42.5) + assert payload["response_cost"] == pytest.approx( + 42.5 * litellm.model_cost["gpt-realtime-translate"]["output_cost_per_second"] + ) + + @pytest.mark.parametrize("profile_prefix", ("us", "global")) @pytest.mark.parametrize( "model_name", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b545a0f090f..bc6fab08c5e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10991,7 +10991,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_6"]; + get: operations["websocket_realtime_websocket_endpoint_get_5"]; put?: never; post?: never; delete?: never; @@ -11064,26 +11064,6 @@ export interface paths { patch?: never; trace?: never; }; - "/openai/v1/realtime/translations": { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - /** - * WebSocket: realtime_websocket_endpoint - * @description WebSocket connection endpoint - */ - get: operations["websocket_realtime_websocket_endpoint_get_3"]; - put?: never; - post?: never; - delete?: never; - options?: never; - head?: never; - patch?: never; - trace?: never; - }; "/openai/v1/responses": { parameters: { query?: never; @@ -13991,7 +13971,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_4"]; + get: operations["websocket_realtime_websocket_endpoint_get_3"]; put?: never; post?: never; delete?: never; @@ -21759,7 +21739,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_5"]; + get: operations["websocket_realtime_websocket_endpoint_get_4"]; put?: never; post?: never; delete?: never; @@ -66833,7 +66813,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_6: { + websocket_realtime_websocket_endpoint_get_5: { parameters: { query?: never; header?: never; @@ -66911,24 +66891,6 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_3: { - parameters: { - query?: never; - header?: never; - path?: never; - cookie?: never; - }; - requestBody?: never; - responses: { - /** @description WebSocket Protocol Switched */ - 101: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - }; - }; responses_api_openai_v1_responses_post: { parameters: { query?: never; @@ -70206,7 +70168,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_4: { + websocket_realtime_websocket_endpoint_get_3: { parameters: { query?: never; header?: never; @@ -70369,7 +70331,7 @@ export interface operations { } | null; transcription?: ({ /** Delay */ - delay?: ("minimal" | "low" | "medium" | "high" | "xhigh") | null; + delay?: string | null; /** Keywords */ keywords?: string[] | null; /** Language */ @@ -70432,7 +70394,7 @@ export interface operations { /** Truncation */ truncation?: unknown | null; /** Type */ - type?: ("realtime" | "transcription" | "translation") | null; + type?: string | null; } & { [key: string]: unknown; }) | null; @@ -81276,7 +81238,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_5: { + websocket_realtime_websocket_endpoint_get_4: { parameters: { query?: never; header?: never; @@ -81439,7 +81401,7 @@ export interface operations { } | null; transcription?: ({ /** Delay */ - delay?: ("minimal" | "low" | "medium" | "high" | "xhigh") | null; + delay?: string | null; /** Keywords */ keywords?: string[] | null; /** Language */ @@ -81502,7 +81464,7 @@ export interface operations { /** Truncation */ truncation?: unknown | null; /** Type */ - type?: ("realtime" | "transcription" | "translation") | null; + type?: string | null; } & { [key: string]: unknown; }) | null; From ae6199d730112ddd27639e5941667b104fb7a57e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:34:56 -0700 Subject: [PATCH 42/45] fix(realtime): address review findings on transcription billing, streaming and realtime auth Per-second pricing replaces token billing only when the priced entry sets no token rates, translation rates fall back per key to the cost map, transcription cache keys include languages and keywords, a stream request a post_call guardrail forces unstreamed comes back as redacted SSE, Azure beta clients are bridged only for GA-only deployments, nested transcription models on the realtime WebSocket are checked against the key, and translation spend carries key and team metadata --- litellm/cost_calculator.py | 40 ++- .../audio_utils/transcription_streaming.py | 23 ++ .../litellm_core_utils/model_param_helper.py | 8 +- .../litellm_core_utils/realtime_streaming.py | 9 +- litellm/llms/azure/realtime/handler.py | 27 +- .../audio_transcription/transformation.py | 46 +++- litellm/main.py | 10 +- ...odel_prices_and_context_window_backup.json | 2 +- litellm/proxy/auth/user_api_key_auth.py | 4 + litellm/proxy/proxy_server.py | 28 ++- litellm/proxy/realtime_endpoints/endpoints.py | 5 +- litellm/realtime_api/main.py | 117 ++++++--- litellm/utils.py | 44 +++- model_prices_and_context_window.json | 2 +- tests/e2e/coverage_registry/schema.py | 1 - .../test_model_param_helper.py | 19 ++ .../test_realtime_streaming.py | 4 +- .../transcriptions/test_gpt_transcribe.py | 42 ++++ .../test_user_api_key_auth_request_flow.py | 52 +++- .../proxy/proxy_server/test_routes_audio.py | 165 ++++++++++++- .../test_realtime_webrtc_endpoints.py | 51 +++- tests/unit/realtime_api/test_main.py | 230 +++++++++++++++++- tests/unit/test_cost_calculator.py | 74 +++++- tests/unit/test_main.py | 23 ++ 24 files changed, 900 insertions(+), 126 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 51251718983..26f837544cd 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -316,6 +316,15 @@ def _get_additional_costs( return None +_TRANSCRIPTION_TOKEN_RATE_KEYS: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", +) +_TRANSCRIPTION_SECOND_RATE_KEYS: Final = ("input_cost_per_second", "output_cost_per_second") + + def _transcription_uses_token_pricing( model: str, custom_llm_provider: str | None, @@ -325,17 +334,9 @@ def _transcription_uses_token_pricing( return False model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) - has_token_pricing: Final = any( - model_info.get(field) - for field in ( - "input_cost_per_token", - "output_cost_per_token", - "input_cost_per_audio_token", - "output_cost_per_audio_token", - ) - ) - if not has_token_pricing and ( - model_info.get("input_cost_per_second") is not None or model_info.get("output_cost_per_second") is not None + if ( + _declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None + and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None ): return False @@ -3687,9 +3688,6 @@ def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[flo return float(input_seconds), float(output_seconds) -_TRANSLATION_RATE_KEYS: Final = ("input_cost_per_second", "output_cost_per_second") - - def handle_realtime_translation_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, @@ -3704,12 +3702,8 @@ def handle_realtime_translation_cost_calculation( for model in (*potential_model_names, litellm_model_name) if model is not None ) - priced_info: Final = next( - (info for info in model_infos if _declared_transcription_rate(info, _TRANSLATION_RATE_KEYS) is not None), - None, - ) - input_cost_per_second: Final = _declared_transcription_rate(priced_info, ("input_cost_per_second",)) or 0.0 - output_cost_per_second: Final = _declared_transcription_rate(priced_info, ("output_cost_per_second",)) or 0.0 + input_cost_per_second: Final = _transcription_rate(("input_cost_per_second",), *model_infos) + output_cost_per_second: Final = _transcription_rate(("output_cost_per_second",), *model_infos) return input_seconds * input_cost_per_second + output_seconds * output_cost_per_second @@ -3776,7 +3770,7 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | return None -def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None: +def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[str, ...]) -> float | None: """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry because ``get_model_info`` synthesizes zero token rates for entries that omit them.""" if info is None: @@ -3790,8 +3784,8 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) ) -def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float: - rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base)) +def _transcription_rate(keys: tuple[str, ...], *infos: ModelInfo | None) -> float: + rates: Final = (_declared_transcription_rate(info, keys) for info in infos) return next((rate for rate in rates if rate is not None), 0.0) diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 1fc18d9eda0..b38b7314663 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -1,6 +1,7 @@ from __future__ import annotations import datetime +import json import traceback from collections.abc import AsyncIterator, Iterator from typing import Final, Protocol @@ -15,6 +16,28 @@ from openai.types.audio import ( from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.types.utils import TranscriptionResponse +TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed" + + +def transcription_stream_requested(value: object) -> bool: + return str(value).lower() in ("1", "true") + + +def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]: + text: Final = response.text or "" + usage: Final = response.usage + languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None)) + done_event: Final = { + "type": "transcript.text.done", + "text": text, + **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}), + **({} if languages is None else {"languages": [language.model_dump() for language in languages]}), + } + return tuple( + f"data: {json.dumps(event, separators=(',', ':'))}\n\n" + for event in ({"type": "transcript.text.delta", "delta": text}, done_event) + ) + class TranscriptionStreamLogging(Protocol): def success_handler( diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index c9c7a0e89ac..c1f231ba141 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -1,6 +1,6 @@ from collections.abc import Mapping from functools import lru_cache -from typing import ClassVar, Final +from typing import ClassVar, Final, get_args from openai.types.chat.completion_create_params import ( CompletionCreateParamsNonStreaming, @@ -20,6 +20,7 @@ from openai.types.responses.response_create_params import ( from litellm._logging import verbose_logger from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams from litellm.types.rerank import RerankRequest @@ -167,6 +168,7 @@ class ModelParamHelper: This follows the OpenAI API Spec """ + litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams)) try: from openai.types.audio.transcription_create_params import ( TranscriptionCreateParamsNonStreaming, @@ -176,11 +178,11 @@ class ModelParamHelper: non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys()) streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys()) - all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs) + all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs) return all_transcription_kwargs except Exception as e: verbose_logger.debug("Error getting transcription kwargs %s", str(e)) - return set() + return litellm_transcription_kwargs @staticmethod def _get_litellm_supported_responses_api_kwargs() -> set[str]: diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 8aa985ed507..66195634efc 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -354,6 +354,7 @@ class RealTimeStreaming: usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, translation_session: bool = False, + bridges_beta_client_to_ga: bool = False, ): self.websocket: _ClientWebSocket = websocket self._replay_scope: Final = client_websocket_scope(websocket) @@ -367,6 +368,7 @@ class RealTimeStreaming: self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] self._is_translation_session = translation_session + self._bridges_beta_client_to_ga: Final = bridges_beta_client_to_ga self._translation_input_seconds = 0.0 self._translation_input_bytes_per_second = 48000.0 self._translation_output_seconds = 0.0 @@ -1143,12 +1145,7 @@ class RealTimeStreaming: return json.dumps(self._session_event_for_client(self._normalize_event_for_ga_client(event))) def _session_event_for_client(self, event: Mapping[str, object]) -> Mapping[str, object]: - if ( - not self._client_wants_beta - or self._backend_uses_beta_protocol - or self._is_translation_session - or event.get("type") not in self._SESSION_EVENT_TYPES - ): + if not self._bridges_beta_client_to_ga or event.get("type") not in self._SESSION_EVENT_TYPES: return event session: Final = _mapping_field(event, "session") return event if session is None else {**event, "session": ga_session_to_beta(session)} diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 4c243d12781..909657dcab3 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -5,6 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ from collections.abc import Mapping +from dataclasses import dataclass from types import MappingProxyType from typing import Final, Protocol, cast @@ -63,23 +64,29 @@ def azure_realtime_requires_ga(model: str) -> bool: return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1 +@dataclass(frozen=True, slots=True) +class AzureRealtimeProtocol: + name: str + bridges_beta_client_to_ga: bool = False + + def azure_realtime_protocol_for_client( configured_protocol: object, *, model: str, + base_model: str | None = None, realtime_mode: str, query_params: RealtimeQueryParams | None, websocket: ScopedWebSocket, -) -> str: +) -> AzureRealtimeProtocol: if isinstance(configured_protocol, str) and configured_protocol: - return configured_protocol - if ( - realtime_mode == "translation" - or (query_params or {}).get("intent") == "transcription" - or azure_realtime_requires_ga(model) - ): - return "GA" - return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" + return AzureRealtimeProtocol(name=configured_protocol) + if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": + return AzureRealtimeProtocol(name="GA") + client_wants_beta: Final = client_sent_openai_beta_realtime_header(websocket) + if azure_realtime_requires_ga(base_model or model): + return AzureRealtimeProtocol(name="GA", bridges_beta_client_to_ga=client_wants_beta) + return AzureRealtimeProtocol(name="beta" if client_wants_beta else "GA") class _ProxyClientWebSocket(Protocol): @@ -196,6 +203,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, realtime_mode: str = "realtime", + bridges_beta_client_to_ga: bool = False, ): import websockets from websockets.asyncio.client import ClientConnection @@ -237,6 +245,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): model if (query_params or {}).get("intent") == "transcription" else None ), translation_session=realtime_mode == "translation", + bridges_beta_client_to_ga=bridges_beta_client_to_ga, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index 872bf447e40..cc22c1f77bb 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,11 +1,24 @@ from abc import ABC, abstractmethod from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Final +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, Final, cast import httpx +from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream +from openai.types.audio import ( + Transcription, + TranscriptionDiarized, + TranscriptionStreamEvent, + TranscriptionVerbose, +) +from openai.types.audio.transcription_create_params import ( + TranscriptionCreateParamsBase, + TranscriptionCreateParamsStreaming, +) from pydantic import BaseModel, TypeAdapter +from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.types.llms.openai import ( @@ -16,6 +29,10 @@ from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse _TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object]) _TRANSCRIPTION_TEXT: Final = TypeAdapter(str) +_TRANSCRIPTION_BODY: Final = TypeAdapter(dict[str, object]) +_TRANSCRIPTION_TYPES_BY_FORMAT: Final = MappingProxyType( + {"verbose_json": TranscriptionVerbose, "diarized_json": TranscriptionDiarized} +) def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]: @@ -58,10 +75,11 @@ def _is_event_stream(http_response: httpx.Response) -> bool: return _content_type(http_response).startswith("text/event-stream") -def _unstreamed_transcription_response(http_response: httpx.Response) -> BaseModel | str: - if _content_type(http_response).startswith("application/json"): - return Transcription.model_validate_json(http_response.content) - return http_response.text +def _unstreamed_transcription_response(http_response: httpx.Response, response_format: object) -> BaseModel | str: + if not _content_type(http_response).startswith("application/json"): + return http_response.text + transcription_type: Final = _TRANSCRIPTION_TYPES_BY_FORMAT.get(str(response_format), Transcription) + return transcription_type.model_construct(None, **_TRANSCRIPTION_BODY.validate_json(http_response.content)) def _non_streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsBase: @@ -79,33 +97,39 @@ def _streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsSt def create_sdk_transcription( client: OpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout ) -> tuple[dict[str, str], Stream[TranscriptionStreamEvent] | BaseModel | str]: - if data.get("stream") is not True: + if not transcription_stream_requested(data.get("stream")): raw_response: Final = client.audio.transcriptions.with_raw_response.create( **_non_streaming_params(data), timeout=timeout ) return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) - raw_stream: Final = client.audio.transcriptions.with_raw_response.create(**_streaming_params(data), timeout=timeout) + raw_stream: Final = client.audio.transcriptions.with_raw_response.create( + **_streaming_params({**data, "stream": True}), timeout=timeout + ) if _is_event_stream(raw_stream.http_response): return dict(raw_stream.headers), raw_stream.parse(to=Stream[TranscriptionStreamEvent]) raw_stream.http_response.read() - return dict(raw_stream.headers), _unstreamed_transcription_response(raw_stream.http_response) + return dict(raw_stream.headers), _unstreamed_transcription_response( + raw_stream.http_response, data.get("response_format") + ) async def acreate_sdk_transcription( client: AsyncOpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout ) -> tuple[dict[str, str], AsyncStream[TranscriptionStreamEvent] | BaseModel | str]: - if data.get("stream") is not True: + if not transcription_stream_requested(data.get("stream")): raw_response: Final = await client.audio.transcriptions.with_raw_response.create( **_non_streaming_params(data), timeout=timeout ) return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) raw_stream: Final = await client.audio.transcriptions.with_raw_response.create( - **_streaming_params(data), timeout=timeout + **_streaming_params({**data, "stream": True}), timeout=timeout ) if _is_event_stream(raw_stream.http_response): return dict(raw_stream.headers), raw_stream.parse(to=AsyncStream[TranscriptionStreamEvent]) await raw_stream.http_response.aread() - return dict(raw_stream.headers), _unstreamed_transcription_response(raw_stream.http_response) + return dict(raw_stream.headers), _unstreamed_transcription_response( + raw_stream.http_response, data.get("response_format") + ) @dataclass diff --git a/litellm/main.py b/litellm/main.py index fef790e7911..e3b6f37cd66 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8214,7 +8214,7 @@ async def atranscription( else: # Call the synchronous function using run_in_executor response = await loop.run_in_executor(None, func_with_context) - if kwargs.get("stream") is True and isinstance(response, AsyncStream): + if isinstance(response, AsyncStream): if file is not None: calculated_duration = calculate_request_duration(file) if calculated_duration is not None: @@ -8569,14 +8569,18 @@ def transcription( # Store duration in _hidden_params for cost calculation without # exposing it in the response body (see sync path comment above). - if response is not None and not isinstance(response, Coroutine): + if ( + response is not None + and not isinstance(response, Coroutine) + and (isinstance(response, (Stream, AsyncStream)) or getattr(response, "duration", None) is None) + ): calculated_duration: Final = calculate_request_duration(file) if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it calculated_duration ) - elif getattr(response, "duration", None) is None and calculated_duration is not None: + elif calculated_duration is not None: response.set_audio_transcription_duration(calculated_duration) if response is None: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1e649b43110..5ae4bcb4ebb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -71200,7 +71200,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "audio" ], diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 07c4626ac23..aaccb288be8 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1181,6 +1181,10 @@ def _apply_budget_limits_to_end_user_params( async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth: + return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model")) + + +async def user_api_key_auth_realtime_websocket(websocket: WebSocket) -> UserAPIKeyAuth: route_path: Final[object] = websocket.scope.get("path") model: Final = resolve_realtime_route_model( model=websocket.query_params.get("model"), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c80c02142a3..2839026552c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -354,6 +354,11 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + TRANSCRIPTION_RAN_UNSTREAMED, + guarded_transcription_events, + transcription_stream_requested, +) from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, @@ -444,6 +449,7 @@ from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # lega user_api_key_auth, user_api_key_auth_accepted_websocket_for_model, user_api_key_auth_live_websocket, + user_api_key_auth_realtime_websocket, user_api_key_auth_websocket, ) from litellm.proxy.batches_endpoints.endpoints import router as batches_router @@ -13390,10 +13396,7 @@ async def audio_transcriptions( try: # Use orjson to parse JSON data, orjson speeds up requests significantly form_data: Final = await get_form_data(request) - data = { - key: value is True or str(value).lower() in ("1", "true") if key == "stream" else value - for key, value in _form_fields_without_file(form_data).items() - } | data + data = {**_form_fields_without_file(form_data), **data} # Include original request and headers in the data data = await add_litellm_data_to_request( @@ -13496,15 +13499,20 @@ async def audio_transcriptions( ) response_headers: Final = {**custom_headers, **(callback_headers or {})} - if data.get("stream") is True and hasattr(response, "__aiter__"): - stream_response: Final = cast(AsyncIterator[TranscriptionStreamEvent], response) + ran_unstreamed: Final = ( + isinstance(response, TranscriptionResponse) + and response.hidden_params.get(TRANSCRIPTION_RAN_UNSTREAMED) is True + ) + if transcription_stream_requested(form_data.get("stream")) and ( + ran_unstreamed or hasattr(response, "__aiter__") + ): async def transcription_event_stream( stream: AsyncIterator[TranscriptionStreamEvent], ) -> AsyncGenerator[str, None]: try: async for event in stream: - yield f"data: {event.model_dump_json()}\n\n" + yield f"data: {event.model_dump_json(exclude_unset=True)}\n\n" except Exception as stream_error: # noqa: BLE001 # stream boundary: a mid-stream failure ends the SSE body as an error event replacement: Final = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -13520,7 +13528,9 @@ async def audio_transcriptions( await close_result return StreamingResponse( - transcription_event_stream(stream_response), + guarded_transcription_events(response) + if isinstance(response, TranscriptionResponse) + else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)), media_type="text/event-stream", headers=response_headers, ) @@ -13685,7 +13695,7 @@ async def realtime_websocket_endpoint( None, description="Comma-separated list of guardrail names to apply to this request.", ), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_realtime_websocket), ): requested_protocols: Final = tuple( protocol.strip() diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 1e57add6b10..eb337826ece 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -211,8 +211,9 @@ async def _prepare_client_secret_session( session_data["type"] = session_type session_model: Final = req.session.model if req.session else None + requested_model: Final = session_model or req.model default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL - model: str = session_model or req.model or default_model + model: str = requested_model or default_model if session_type != "transcription": await can_key_call_resolved_model( model=model, @@ -220,7 +221,7 @@ async def _prepare_client_secret_session( llm_model_list=llm_model_list, llm_router=llm_router, ) - if session_data is not None: + if session_data is not None and (requested_model or session_type == "translation"): session_data["model"] = model if session_type == "translation": await _authorize_and_bind_nested_transcription_models( diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 53d53237773..3d907daec44 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -2,6 +2,7 @@ import asyncio import os +import traceback from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType @@ -15,9 +16,11 @@ import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, + REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) +from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig @@ -85,24 +88,30 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> return MappingProxyType({**credential_values, **model_params}) -def with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: - if session.get("type") == "transcription": - audio: Final = as_str_mapping(session.get("audio")) or {} - audio_input: Final = as_str_mapping(audio.get("input")) or {} - transcription: Final = as_str_mapping(audio_input.get("transcription")) or {} - return { - **session, +def _transcription_model_overrides(session: Mapping[str, object], model_name: str) -> Mapping[str, object]: + flat_transcription: Final = as_str_mapping(session.get("input_audio_transcription")) + audio: Final = as_str_mapping(session.get("audio")) or {} + audio_input: Final = as_str_mapping(audio.get("input")) or {} + nested_transcription: Final = as_str_mapping(audio_input.get("transcription")) + flat_part: Final = ( + {} if flat_transcription is None else {"input_audio_transcription": {**flat_transcription, "model": model_name}} + ) + nested_part: Final = ( + {} + if nested_transcription is None and flat_transcription is not None + else { "audio": { **audio, - "input": { - **audio_input, - "transcription": { - **transcription, - "model": model_name, - }, - }, - }, + "input": {**audio_input, "transcription": {**(nested_transcription or {}), "model": model_name}}, + } } + ) + return MappingProxyType({**flat_part, **nested_part}) + + +def with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: + if session.get("type") == "transcription": + return {**session, **_transcription_model_overrides(session, model_name)} if "model" not in session: return session return {**session, "model": model_name} @@ -121,6 +130,17 @@ def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]: return {**policy_pipeline_state, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})} +_GUARDRAIL_OPT_OUT_METADATA_KEYS: Final = ("user_api_key_metadata", "user_api_key_team_metadata") + + +def _translation_litellm_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + request_metadata: Final = _METADATA_ADAPTER.validate_python(kwargs.get("metadata") or {}) + guardrail_opt_outs: Final = { + key: request_metadata[key] for key in _GUARDRAIL_OPT_OUT_METADATA_KEYS if key in request_metadata + } + return MappingProxyType({**guardrail_opt_outs, **_build_litellm_metadata(kwargs)}) + + _HTTP_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) _WEBSOCKET_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, *AZURE_OPENAI_AUDIO_PROVIDERS}) @@ -140,11 +160,11 @@ _TranslationRejection = _TranslationProviderUnsupported | _TranslationGuardrails def _translation_rejection( - custom_llm_provider: str, supported_providers: frozenset[str], kwargs: Mapping[str, object] + custom_llm_provider: str, supported_providers: frozenset[str], litellm_metadata: Mapping[str, object] ) -> _TranslationRejection | None: if custom_llm_provider not in supported_providers: return _TranslationProviderUnsupported(provider=custom_llm_provider, supported_providers=supported_providers) - if realtime_guardrails_apply({"litellm_metadata": _build_litellm_metadata(kwargs)}): + if realtime_guardrails_apply({"litellm_metadata": dict(litellm_metadata)}): return _TranslationGuardrailsUnsupported() return None @@ -172,19 +192,37 @@ def _translation_rejection_error_type(rejection: _TranslationRejection) -> str: assert_never(rejection) -def _raise_if_http_translation_rejected(custom_llm_provider: str, model: str, kwargs: Mapping[str, object]) -> None: - rejection: Final = _translation_rejection(custom_llm_provider, _HTTP_TRANSLATION_PROVIDERS, kwargs) - if rejection is None: - return - raise litellm.BadRequestError( +def _translation_rejection_error( + rejection: _TranslationRejection, model: str, custom_llm_provider: str +) -> BadRequestError: + return BadRequestError( message=_translation_rejection_message(rejection), model=model, llm_provider=custom_llm_provider ) -async def _refuse_translation_session(websocket: "WebSocket", rejection: _TranslationRejection) -> None: +def _raise_if_http_translation_rejected(custom_llm_provider: str, model: str, kwargs: Mapping[str, object]) -> None: + rejection: Final = _translation_rejection( + custom_llm_provider, _HTTP_TRANSLATION_PROVIDERS, _translation_litellm_metadata(kwargs) + ) + if rejection is None: + return + raise _translation_rejection_error(rejection, model, custom_llm_provider) + + +async def _refuse_translation_session( + websocket: "WebSocket", + rejection: _TranslationRejection, + logging_obj: LiteLLMLogging, + model: str, + custom_llm_provider: str, +) -> None: message: Final = _translation_rejection_message(rejection) await websocket.send_text(realtime_error_event(message, error_type=_translation_rejection_error_type(rejection))) await websocket.close(code=1008, reason=websocket_close_reason(message, fallback="Translation session refused")) + error: Final = _translation_rejection_error(rejection, model, custom_llm_provider) + await logging_obj.dispatch_failure_handlers( + error, "".join(traceback.format_exception(error)), prefer_async_handlers=True + ) def _get_realtime_http_provider_config( @@ -793,13 +831,21 @@ async def _arealtime( custom_llm_provider=_custom_llm_provider, ) + session_litellm_metadata: Final = ( + dict(_translation_litellm_metadata(kwargs)) + if realtime_mode == "translation" + else _build_litellm_metadata(kwargs) + ) translation_rejection: Final = ( - _translation_rejection(_custom_llm_provider, _WEBSOCKET_TRANSLATION_PROVIDERS, kwargs) + _translation_rejection(_custom_llm_provider, _WEBSOCKET_TRANSLATION_PROVIDERS, session_litellm_metadata) if realtime_mode == "translation" else None ) if translation_rejection is not None: - await _refuse_translation_session(websocket, translation_rejection) + await _refuse_translation_session( + websocket, translation_rejection, litellm_logging_obj, model, _custom_llm_provider + ) + litellm_logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] = True return if live_session_start is not None: @@ -811,7 +857,7 @@ async def _arealtime( session_start=live_session_start, api_base=dynamic_api_base or litellm_params.api_base, api_key=dynamic_api_key, - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, litellm_params=kwargs, ) return @@ -836,7 +882,7 @@ async def _arealtime( timeout=timeout, headers=headers, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, query_params=query_params, ) elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS: @@ -859,9 +905,15 @@ async def _arealtime( GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), ) ) + configured_base_model: Final = (as_str_mapping(litellm_params_dict) or {}).get("base_model") realtime_protocol: Final = azure_realtime_protocol_for_client( configured_realtime_protocol, model=model, + base_model=( + configured_base_model.removeprefix(f"{_custom_llm_provider}/") + if isinstance(configured_base_model, str) + else None + ), query_params=query_params, realtime_mode=realtime_mode, websocket=websocket, @@ -876,11 +928,12 @@ async def _arealtime( client=None, timeout=timeout, logging_obj=litellm_logging_obj, - realtime_protocol=realtime_protocol, + realtime_protocol=realtime_protocol.name, query_params=query_params, realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, + bridges_beta_client_to_ga=realtime_protocol.bridges_beta_client_to_ga, ) elif _custom_llm_provider == "openai": api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/" @@ -899,7 +952,7 @@ async def _arealtime( query_params=query_params, realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, ) elif _custom_llm_provider == "bedrock": # Extract AWS parameters from kwargs @@ -954,7 +1007,7 @@ async def _arealtime( timeout=timeout, query_params=query_params, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, ) elif _custom_llm_provider == "vertex_ai": vertex_credentials: Final = ( @@ -1014,7 +1067,7 @@ async def _arealtime( timeout=timeout, headers=headers, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, query_params=query_params, ) else: diff --git a/litellm/utils.py b/litellm/utils.py index 11ee0fb56c8..032192a2504 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -996,6 +996,26 @@ def _is_streaming_response_for_correlation(result: object) -> bool: return isinstance(result, CustomStreamWrapper) +def _returns_requested_stream(kwargs: dict[str, object], call_type: CallTypes | str, result: object) -> bool: + if call_type in (CallTypes.transcription, CallTypes.atranscription): + return isinstance(result, (openai.Stream, openai.AsyncStream)) + return _is_streaming_request(kwargs=kwargs, call_type=call_type) + + +def _transcription_runs_unstreamed(call_type: str, call_kwargs: Mapping[str, object]) -> bool: + return call_type == CallTypes.atranscription.value and call_kwargs.get("stream") is False + + +def _marked_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import TRANSCRIPTION_RAN_UNSTREAMED + + if not runs_unstreamed or not isinstance(result, TranscriptionResponse): + return result + marked: Final = result.model_copy() + marked.hidden_params = {**result.hidden_params, TRANSCRIPTION_RAN_UNSTREAMED: True} + return marked + + def _is_converted_stream_result(result: object) -> bool: from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -1664,7 +1684,13 @@ async def async_pre_call_deployment_hook(kwargs: dict[str, Any], call_type: str) def _unstreamed_when_post_call_guardrails_apply( kwargs: dict[str, object], call_type: CallTypes | None ) -> dict[str, object]: - if call_type is not CallTypes.atranscription or kwargs.get("stream") is not True: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested + + if ( + call_type is not CallTypes.atranscription + or not transcription_stream_requested(kwargs.get("stream")) + or _deployment_provider(kwargs) not in (LlmProviders.OPENAI.value, LlmProviders.AZURE.value) + ): return kwargs unstreamed: Final = {**kwargs, "stream": False} custom_guardrail_class: Final = _get_cached_custom_guardrail() @@ -2211,6 +2237,7 @@ def client(original_function): kwargs["litellm_logging_obj"] = logging_obj modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type) + transcription_runs_unstreamed: Final = _transcription_runs_unstreamed(call_type, modified_kwargs) if modified_kwargs is not None: kwargs = modified_kwargs @@ -2261,7 +2288,9 @@ def client(original_function): if _is_converted_stream_result(_caching_handler_response.cached_result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True - return _caching_handler_response.cached_result + return _marked_when_run_unstreamed( + _caching_handler_response.cached_result, transcription_runs_unstreamed + ) elif _caching_handler_response.embedding_all_elements_cache_hit is True: return _caching_handler_response.final_embedding_cached_response @@ -2369,10 +2398,13 @@ def client(original_function): # Only run if call_type is a valid value in CallTypes _call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) if _call_type_enum is not None: - result = await async_post_call_success_deployment_hook( - request_data=call_kwargs, - response=result, - call_type=_call_type_enum, + result = _marked_when_run_unstreamed( + await async_post_call_success_deployment_hook( + request_data=call_kwargs, + response=result, + call_type=_call_type_enum, + ), + transcription_runs_unstreamed, ) ## Add response to cache diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1e649b43110..5ae4bcb4ebb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -71200,7 +71200,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", - "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_modalities": [ "audio" ], diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 47df7dae635..7d9bc380bd0 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -93,7 +93,6 @@ LlmCapability = Literal[ "tool_search_history", "tool_use", "upstream_stream_failure", - "transcription", "translation", "vision", diff --git a/tests/unit/litellm_core_utils/test_model_param_helper.py b/tests/unit/litellm_core_utils/test_model_param_helper.py index 6293c748505..df37139bc7b 100644 --- a/tests/unit/litellm_core_utils/test_model_param_helper.py +++ b/tests/unit/litellm_core_utils/test_model_param_helper.py @@ -1,5 +1,9 @@ +from collections.abc import Sequence +from typing import Final +import pytest +from litellm.caching.caching import Cache from litellm.litellm_core_utils.model_param_helper import ModelParamHelper @@ -46,3 +50,18 @@ def test_langfuse_model_parameters_no_secret_leakage(): assert "secret_fields" not in sanitized assert "authorization" not in sanitized assert "headers" not in sanitized + + +@pytest.mark.parametrize( + "hint,first_value,second_value", + [("languages", ("en",), ("fr",)), ("keywords", ("LiteLLM",), ("Kubernetes",))], +) +def test_transcription_cache_key_separates_recognition_hints( + hint: str, first_value: Sequence[str], second_value: Sequence[str] +) -> None: + cache: Final = Cache() + request: Final = {"model": "gpt-4o-transcribe", "file": "meeting.wav", "response_format": "json"} + + assert cache.get_cache_key(**request, **{hint: first_value}) != cache.get_cache_key( + **request, **{hint: second_value} + ) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 320b35ff292..729e5bb1576 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -551,7 +551,9 @@ async def test_beta_client_bridged_to_a_ga_backend_receives_the_beta_session_sha logging_obj: Final = _mock_logging() logging_obj.async_success_handler = AsyncMock() logging_obj.success_handler = MagicMock() - streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False) + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False, bridges_beta_client_to_ga=True + ) await streaming.backend_to_client_send_messages() diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 4963c5f9a20..613ab467acb 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -545,6 +545,48 @@ def _duration_priced_transcript(streamed: bool) -> httpx.Response: ) +def test_whisper_stream_request_answered_with_json_keeps_the_unstreamed_response_shape( + local_model_cost_map: None, +) -> None: + verbose_transcript: Final = { + "task": "transcribe", + "language": "english", + "duration": 1.0, + "text": "hello world", + "usage": {"type": "duration", "seconds": 1}, + } + with _openai_transcription_client( + httpx.MockTransport(lambda request: httpx.Response(200, json=verbose_transcript)) + ) as client: + unstreamed: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + ignored_stream: Final = litellm.transcription( + model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client + ) + + assert isinstance(ignored_stream, TranscriptionResponse) + assert json.loads(ignored_stream.model_dump_json()) == json.loads(unstreamed.model_dump_json()) + + +@pytest.mark.parametrize("requested_stream", ("true", "True", "1")) +def test_transcription_stream_requested_as_a_form_string_streams_like_a_bool( + local_model_cost_map: None, requested_stream: str +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=_asks_for_stream(sent_bodies[-1])) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=requested_stream, client=client + ) + events: Final = [event.type for event in stream] + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert len(sent_bodies) == 1 + + def _asks_for_stream(request_body: bytes) -> bool: return b'name="stream"\r\n\r\ntrue' in request_body diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 373636f7a8c..cb71cfd2440 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -11945,7 +11945,7 @@ async def test_translation_websocket_auth_enforces_the_default_model_budget( from fastapi import WebSocket from litellm.proxy import proxy_server - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket model: Final = "gpt-realtime-translate" token: Final = UserAPIKeyAuth( @@ -11980,7 +11980,7 @@ async def test_translation_websocket_auth_enforces_the_default_model_budget( ) with pytest.raises(HTTPException) as denied: - await user_api_key_auth_websocket(websocket) + await user_api_key_auth_realtime_websocket(websocket) assert denied.value.status_code == 403 assert model in str(denied.value.detail) @@ -12041,7 +12041,7 @@ async def test_managed_agent_transcription_websocket_without_a_model_is_refused( from fastapi import WebSocket from litellm.proxy import proxy_server - from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket from litellm.types.agents import AgentResponse token: Final = UserAPIKeyAuth(token="test-key") @@ -12068,12 +12068,56 @@ async def test_managed_agent_transcription_websocket_without_a_model_is_refused( ) with pytest.raises(HTTPException) as denied: - await user_api_key_auth_websocket(websocket) + await user_api_key_auth_realtime_websocket(websocket) assert denied.value.status_code == 403 assert "Managed inference requires an explicit or configured model" in str(denied.value.detail) +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/openai_passthrough/v1/realtime", "/v1/responses", "/vertex_ai/live")) +async def test_non_realtime_websocket_auth_checks_no_default_transcription_model( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + + exhausted_model: Final = "gpt-realtime-whisper" + token: Final = UserAPIKeyAuth( + token="test-key", + model_max_budget={exhausted_model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={exhausted_model: 10.0}) + with pytest.raises(litellm.BudgetExceededError): + await limiter.is_key_within_model_budget(user_api_key_dict=token, model=exhausted_model) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": b"intent=transcription", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + assert await user_api_key_auth_websocket(websocket) is token + + @pytest.mark.asyncio async def test_admission_and_budget_reservation_read_the_key_spend_counter_with_one_redis_mget(): from fastapi import Request diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index dd9e87c7964..e0a228b6eea 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -21,8 +21,10 @@ import pytest import respx from fastapi.testclient import TestClient from openai import AsyncOpenAI +from starlette.datastructures import FormData, UploadFile import litellm +from litellm.caching.caching import Cache from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server from litellm.proxy.route_llm_request import route_request @@ -167,7 +169,7 @@ def patched_transcription_error(monkeypatch, patched_transcription): @pytest.fixture def patched_transcription_stream(monkeypatch, patched_transcription): class _FakeEvent: - def model_dump_json(self): + def model_dump_json(self, exclude_unset: bool = False) -> str: return '{"type":"transcript.text.done","text":"hello world"}' class _FakeAsyncStream: @@ -424,7 +426,11 @@ def _streams_when_asked(request: httpx.Request) -> httpx.Response: content=b'data: {"type":"transcript.text.done","text":"my pin is 1234"}\n\n', headers={"content-type": "text/event-stream"}, ) - return httpx.Response(200, json={"text": "my pin is 1234"}) + return httpx.Response(200, json={"text": "my pin is 1234", "usage": {"type": "duration", "seconds": 2}}) + + +def _sse_events(body: str) -> tuple[object, ...]: + return tuple(json.loads(frame.removeprefix("data: ")) for frame in body.split("\n\n") if frame) @pytest.mark.usefixtures("patched_transcription_stream") @@ -471,13 +477,164 @@ def test_streamed_transcription_honors_a_deployment_post_call_guardrail( assert response.status_code == 200 if deployment_guardrails: - assert response.headers["content-type"].startswith("application/json") - assert response.json()["text"] == "[REDACTED]" + assert response.headers["content-type"].startswith("text/event-stream") + assert _sse_events(response.text) == ( + {"type": "transcript.text.delta", "delta": "[REDACTED]"}, + {"type": "transcript.text.done", "text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}, + ) else: assert response.headers["content-type"].startswith("text/event-stream") assert '"text":"my pin is 1234"' in response.text +@pytest.mark.usefixtures("patched_transcription_stream") +def test_cached_guarded_transcription_still_streams_to_a_stream_client( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + upstream_route: Final = respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock( + side_effect=_streams_when_asked + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + "guardrails": ["transcript-redactor"], + }, + } + ], + cache_responses=True, + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + responses: Final = [ + client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + for _ in range(2) + ] + + assert upstream_route.call_count == 1 + assert [response.headers["content-type"].split(";")[0] for response in responses] == ["text/event-stream"] * 2 + assert _sse_events(responses[1].text)[-1] == { + "type": "transcript.text.done", + "text": "[REDACTED]", + "usage": {"type": "duration", "seconds": 2}, + } + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_frames_keep_the_provider_event_shape( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider_events: Final = ( + {"type": "transcript.text.delta", "delta": "hello"}, + {"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 2}}, + ) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + content="".join(f"data: {json.dumps(event)}\n\n" for event in provider_events).encode(), + headers={"content-type": "text/event-stream"}, + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription( + model="openai/gpt-transcribe", file=data["file"], stream=data["stream"], client=upstream + ) + + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert _sse_events(response.text) == provider_events + + +async def _scribe_stream_form(request: object) -> FormData: + return FormData( + [ + ("file", UploadFile(filename="sample.wav", file=io.BytesIO(b"audio"))), + ("model", "scribe"), + ("stream", "true"), + ] + ) + + +@pytest.mark.usefixtures("patched_transcription") +def test_transcription_stream_form_field_reaches_other_providers_as_sent( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = respx_mock.post("https://elevenlabs.example/v1/speech-to-text").mock( + return_value=httpx.Response(200, json={"text": "hello world", "language_code": "en"}) + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "scribe", + "litellm_params": { + "model": "elevenlabs/scribe_v1", + "api_key": "sk-test", + "api_base": "https://elevenlabs.example", + }, + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + monkeypatch.setattr(proxy_server, "get_form_data", _scribe_stream_form) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "scribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert response.json()["text"] == "hello world" + assert b'name="stream"\r\n\r\ntrue\r\n' in upstream.calls.last.request.content + + def _streams_chat_and_transcription(request: httpx.Request) -> httpx.Response: sse_body: Final = ( b'data: {"id":"chat-1","object":"chat.completion.chunk","created":0,"model":"gpt-4o",' diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 8eada379418..2e3e3d4f7d7 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -1613,6 +1613,7 @@ def _post_through_the_azure_deployments( monkeypatch: pytest.MonkeyPatch, path: str, body: object, + deployments: tuple[tuple[str, str], ...] = _AZURE_REALTIME_DEPLOYMENTS, ) -> tuple[httpx.Response, respx.Route]: from litellm.proxy import proxy_server @@ -1626,7 +1627,7 @@ def _post_through_the_azure_deployments( "api_key": "azure-test-key", }, } - for alias, deployment in _AZURE_REALTIME_DEPLOYMENTS + for alias, deployment in deployments ] monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) monkeypatch.setattr(proxy_server, "llm_model_list", model_list) @@ -1636,7 +1637,7 @@ def _post_through_the_azure_deployments( monkeypatch.setitem( proxy_app.dependency_overrides, user_api_key_auth, - lambda: UserAPIKeyAuth(user_id="test-user", models=[alias for alias, _ in _AZURE_REALTIME_DEPLOYMENTS]), + lambda: UserAPIKeyAuth(user_id="test-user", models=[alias for alias, _ in deployments]), ) with ( patch( # test-quality-ok: endpoint test isolates request metadata enrichment @@ -1783,6 +1784,52 @@ def test_client_secrets_forward_every_session_value_the_caller_set( assert _upstream_session_value(upstream, session_path) == expected +def test_client_secrets_without_a_model_let_the_upstream_pick_its_default_through_a_wildcard_deployment( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + {"session": {"type": "realtime"}}, + deployments=(("*", "openai/*"),), + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content)["session"] == {"type": "realtime"} + + +def test_client_secrets_send_the_deployment_model_in_the_flat_transcription_field( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + {"session": {"type": "transcription", "input_audio_transcription": {"model": "azure-transcribe-alias"}}}, + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content)["session"] == { + "type": "transcription", + "input_audio_transcription": {"model": "qa-whisper-deployment"}, + } + + @pytest.mark.parametrize( ("path", "body"), ( diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index a0cdadff982..44c469f6ed7 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -21,9 +21,12 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID from openai import AsyncOpenAI +from websockets.exceptions import ConnectionClosed import litellm +from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.realtime_streaming import TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, RealTimeStreaming from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -905,7 +908,7 @@ def test_azure_realtime_protocol_defaults_to_ga_when_none_is_configured( None, model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_BETA_CLIENT ) - assert protocol == "GA" + assert protocol.name == "GA" @pytest.mark.parametrize(("model", "realtime_mode", "query_params"), _GA_DEFAULT_SESSIONS) @@ -918,7 +921,7 @@ def test_azure_realtime_protocol_configured_by_the_operator_wins( "beta", model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_GA_CLIENT ) - assert protocol == "beta" + assert protocol.name == "beta" async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: @@ -968,6 +971,134 @@ async def test_arealtime_azure_env_beta_protocol_wins_over_a_ga_client(monkeypat ) +class _BetaClientThatStaysConnected: + def __init__(self) -> None: + self.scope: Final = {"headers": ((b"openai-beta", b"realtime=v1"),)} + self.sent: tuple[str, ...] = () + + async def receive_text(self) -> str: + return await asyncio.get_running_loop().create_future() + + async def send_text(self, data: str) -> None: + self.sent = (*self.sent, data) + + async def close(self, code: int = 1000, reason: str = "") -> None: + return None + + +class _ConnectToABackendThatSendsOneFrame: + def __init__(self, frame: Mapping[str, object]) -> None: + self.url: str | None = None + self._backend: Final = MagicMock(recv=AsyncMock(side_effect=[json.dumps(frame), ConnectionClosed(None, None)])) + + def __call__( + self, url: str, *, additional_headers: Mapping[str, str], max_size: int, ssl: object + ) -> "_ConnectToABackendThatSendsOneFrame": + self.url = url + return self + + async def __aenter__(self) -> MagicMock: + return self._backend + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + return None + + +def _realtime_websocket_logging(*failure_callbacks: CustomLogger) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-realtime-websocket", + function_id="test-realtime-websocket", + dynamic_async_failure_callbacks=list(failure_callbacks), + ) + + +_GA_SESSION_CREATED: Final = { + "type": "session.created", + "event_id": "event_1", + "session": { + "type": "realtime", + "id": "sess_1", + "model": "my-realtime-deployment", + "output_modalities": ["text"], + "audio": {"output": {"voice": "marin"}}, + }, +} + + +async def _azure_session_created_seen_by_a_beta_client( + session_kwargs: Mapping[str, object], +) -> tuple[str | None, object]: + client: Final = _BetaClientThatStaysConnected() + connect: Final = _ConnectToABackendThatSendsOneFrame(_GA_SESSION_CREATED) + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="azure/my-realtime-deployment", + websocket=client, + api_base="https://my-endpoint.openai.azure.com", + api_key="fake-key", + litellm_logging_obj=_realtime_websocket_logging(), + **session_kwargs, + ) + return connect.url, json.loads(client.sent[0]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("session_kwargs", "expected_url"), + ( + pytest.param( + {"realtime_protocol": "GA"}, + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + id="operator-set-ga", + ), + pytest.param( + {"query_params": {"intent": "transcription"}}, + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?intent=transcription", + id="transcription-intent", + ), + ), +) +async def test_arealtime_azure_beta_client_on_a_ga_session_it_did_not_need_bridging_for_keeps_ga_session_events( + monkeypatch: pytest.MonkeyPatch, session_kwargs: Mapping[str, object], expected_url: str +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + + assert await _azure_session_created_seen_by_a_beta_client(session_kwargs) == (expected_url, _GA_SESSION_CREATED) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "base_model_kwargs", + ( + pytest.param({"base_model": "azure/gpt-realtime-2.1"}, id="base-model-param"), + pytest.param({"metadata": {"model_info": {"base_model": "azure/gpt-realtime-2.1"}}}, id="model-info"), + ), +) +async def test_arealtime_azure_bridges_a_beta_client_to_ga_when_the_deployment_base_model_is_ga_only( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, base_model_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + + assert await _azure_session_created_seen_by_a_beta_client(base_model_kwargs) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + { + "type": "session.created", + "event_id": "event_1", + "session": {"id": "sess_1", "model": "my-realtime-deployment", "modalities": ["text"], "voice": "marin"}, + }, + ) + + async def _vertex_provider_config_for(monkeypatch, model: str, vertex_location: str | None): from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import VertexChirpRealtimeConfig from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig @@ -1462,6 +1593,7 @@ def _realtime_http_route_call( custom_llm_provider: str, api_base: str, client: AsyncHTTPHandler | None = None, + metadata: Mapping[str, object] | None = None, ) -> Awaitable[httpx.Response]: if route == "client_secrets": return realtime_main.acreate_realtime_client_secret.__wrapped__( @@ -1480,6 +1612,7 @@ def _realtime_http_route_call( api_base=api_base, client=client, litellm_logging_obj=logging_obj, + metadata=metadata, ) if route == "calls": return realtime_main.arealtime_calls.__wrapped__( @@ -1499,6 +1632,7 @@ def _realtime_http_route_call( api_base=api_base, client=client, litellm_logging_obj=logging_obj, + metadata=metadata, ) @@ -1757,6 +1891,38 @@ async def test_translation_http_routes_refuse_a_guarded_request_before_any_upstr assert upstream.called is False +_DEFAULT_ON_GUARDRAIL_OPT_OUTS: Final = ( + pytest.param({"user_api_key_metadata": {"disable_global_guardrails": True}}, id="key-disables-global"), + pytest.param( + {"user_api_key_team_metadata": {"opted_out_global_guardrails": ["translation-blocker"]}}, id="team-opts-out" + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "path"), + (("translation_client_secrets", "client_secrets"), ("translation_calls", "calls")), +) +@pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +async def test_translation_http_routes_serve_a_key_opted_out_of_the_default_on_guardrail( + route: str, path: str, key_metadata: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + _block_every_request_with_a_guardrail(monkeypatch) + gateway: Final = _GatewayThatRecordsTheUrl() + + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + metadata=key_metadata, + ) + + assert gateway.url == f"https://upstream.example/v1/realtime/translations/{path}" + + @pytest.mark.asyncio @pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) async def test_translation_http_routes_refuse_a_provider_other_than_openai_and_azure_before_any_upstream_call( @@ -1791,7 +1957,7 @@ async def test_translation_websocket_closes_1008_for_a_guarded_request_before_th websocket=websocket, api_key="fake-key", realtime_mode="translation", - litellm_logging_obj=FakeLogging(), + litellm_logging_obj=_realtime_websocket_logging(), ) assert connect.url is None @@ -1801,6 +1967,62 @@ async def test_translation_websocket_closes_1008_for_a_guarded_request_before_th ] +@pytest.mark.asyncio +@pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +async def test_translation_websocket_opens_for_a_key_opted_out_of_the_default_on_guardrail( + key_metadata: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + _block_every_request_with_a_guardrail(monkeypatch) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + metadata=key_metadata, + ) + + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + + +class _FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[object, ...] = () + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures = (*self.failures, kwargs.get("exception")) + + +@pytest.mark.asyncio +async def test_translation_websocket_refusal_logs_the_session_as_a_failure(monkeypatch: pytest.MonkeyPatch) -> None: + _block_every_request_with_a_guardrail(monkeypatch) + recorder: Final = _FailureRecorder() + logging_obj: Final = _realtime_websocket_logging(recorder) + + with patch("websockets.connect", _CapturingConnect()): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=_RecordingClientWebSocket(), + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=logging_obj, + ) + + assert [(type(failure), getattr(failure, "status_code", None)) for failure in recorder.failures] == [ + (litellm.BadRequestError, 400) + ] + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in str(recorder.failures[0]) + assert logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] is True + + class _NativeLifecycleGuardrail(CustomGuardrail): use_native_lifecycle_hooks: ClassVar[bool] = True @@ -1845,7 +2067,7 @@ async def test_translation_websocket_closes_1008_for_a_provider_other_than_opena websocket=websocket, api_key="fake-key", realtime_mode="translation", - litellm_logging_obj=FakeLogging(), + litellm_logging_obj=_realtime_websocket_logging(), ) assert connect.url is None diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index c23f30f3382..84763b58351 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -6957,6 +6957,56 @@ def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypat assert cost == pytest.approx(0.000075 * 2) +@pytest.mark.parametrize( + "deployment_id,deployment_rates,expected_cost", + [ + ("per-second-only", {"input_cost_per_second": 0.0001}, 30 * 0.0001), + ("zero-token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 0.0}, 0.0), + ("token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 1e-6}, 500 * 1e-6), + ], +) +def test_transcription_deployment_bills_seconds_only_without_a_declared_token_rate( + monkeypatch: pytest.MonkeyPatch, + deployment_id: str, + deployment_rates: Mapping[str, float], + expected_cost: float, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {**deployment_rates, "litellm_provider": "openai", "mode": "audio_transcription"}, + ) + transcription: Final = convert_to_model_response_object( + response_object={ + "text": "What is the weather in Paris?", + "usage": { + "type": "tokens", + "input_tokens": 500, + "output_tokens": 100, + "total_tokens": 600, + "input_token_details": {"audio_tokens": 500, "text_tokens": 0}, + }, + }, + model_response_object=TranscriptionResponse(), + hidden_params={ + "model": "gpt-4o-transcribe", + "custom_llm_provider": "openai", + "audio_transcription_duration": 30.0, + }, + response_type="audio_transcription", + ) + + cost: Final = completion_cost( + completion_response=transcription, + model="openai/gpt-4o-transcribe", + call_type="atranscription", + custom_pricing=True, + router_model_id=deployment_id, + ) + + assert cost == pytest.approx(expected_cost) + + def test_gpt_image_2_azure_cost_tracking(): azure_image_generation_response: Final = { "created": 1758585600, @@ -10017,7 +10067,19 @@ def _translation_session_cost(deployment_id: str) -> float: ) -def test_realtime_translation_deployment_per_second_rate_replaces_both_public_rates(_local_model_cost_map) -> None: +@pytest.mark.parametrize( + "deployment_id,deployment_rates,bills_public_output_rate", + [ + ("translate-input-priced", {"input_cost_per_second": 0.001}, True), + ("translate-free-output", {"input_cost_per_second": 0.001, "output_cost_per_second": 0.0}, False), + ], +) +def test_realtime_translation_deployment_per_second_rates_fall_back_to_public_rates_per_key( + _local_model_cost_map: None, + deployment_id: str, + deployment_rates: Mapping[str, float], + bills_public_output_rate: bool, +) -> None: litellm.Router( model_list=[ { @@ -10025,14 +10087,18 @@ def test_realtime_translation_deployment_per_second_rate_replaces_both_public_ra "litellm_params": { "model": "openai/gpt-realtime-translate", "api_key": "sk-test", - "input_cost_per_second": 0.001, + **deployment_rates, }, - "model_info": {"id": "translate-input-priced"}, + "model_info": {"id": deployment_id}, } ] ) + public_output_rate: Final = litellm.model_cost["gpt-realtime-translate"]["output_cost_per_second"] - assert _translation_session_cost("translate-input-priced") == pytest.approx(60 * 0.001) + assert public_output_rate > 0 + assert _translation_session_cost(deployment_id) == pytest.approx( + 60 * 0.001 + (60 * public_output_rate if bills_public_output_rate else 0.0) + ) def test_realtime_translation_deployment_without_per_second_rates_bills_public_rates(_local_model_cost_map) -> None: diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 4a2899c4ef1..f2dc100a0dc 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5070,6 +5070,29 @@ async def test_groq_atranscription_honors_base_url_alias(respx_mock: respx.MockR assert response.text == "hello" +@pytest.mark.parametrize( + "provider_body,expected_upload_measurements", + [({"text": "hello", "duration": 2.5}, 0), ({"text": "hello"}, 1)], + ids=["provider_reports_duration", "provider_omits_duration"], +) +def test_transcription_measures_the_upload_only_when_the_provider_omits_the_duration( + respx_mock: respx.MockRouter, provider_body: Mapping[str, object], expected_upload_measurements: int +): + respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/transcriptions").mock( + return_value=httpx.Response(200, json=provider_body) + ) + + with patch.object( + litellm_main, "calculate_request_duration", wraps=litellm_main.calculate_request_duration + ) as measure_upload: + response: Final = litellm.transcription( + model="groq/whisper-large-v3", file=GROQ_WAV_FILE, base_url=GROQ_INTERNAL_BASE, api_key="fake-key" + ) + + assert response.text == "hello" + assert measure_upload.call_count == expected_upload_measurements + + @pytest.mark.parametrize("model_name", ("whisper-1", "gpt-4o-transcribe", "gpt-transcribe")) def test_openai_compatible_transcription_with_an_openai_model_name_reaches_the_server( respx_mock: respx.MockRouter, model_name: str From c48cbde7b7f2bf434ab15abed769532dda06debd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:20:12 -0700 Subject: [PATCH 43/45] fix(azure): pick the GA realtime protocol for deployments whose model_info.base_model is GA-only --- litellm/realtime_api/main.py | 16 +++--- tests/unit/realtime_api/test_main.py | 75 ++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 3d907daec44..b7f99992593 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -38,7 +38,7 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ProviderConfigManager, get_base_model_from_metadata, resolve_api_key_secret_reference from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -462,6 +462,11 @@ def _with_upstream_transcription_model(request_data: Mapping[str, object], model } +def _deployment_base_model(kwargs: Mapping[str, object], custom_llm_provider: str) -> str | None: + base_model: Final = get_base_model_from_metadata(model_call_details={"litellm_params": kwargs}) + return None if base_model is None else base_model.removeprefix(f"{custom_llm_provider}/") + + @wrapper_client async def acreate_realtime_transcription_session( model: str | None = None, @@ -534,7 +539,7 @@ async def acreate_realtime_transcription_session( logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, provider_config=provider_config, - model=model_name, + model=_deployment_base_model(kwargs, custom_llm_provider) or model_name, extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, @@ -905,15 +910,10 @@ async def _arealtime( GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), ) ) - configured_base_model: Final = (as_str_mapping(litellm_params_dict) or {}).get("base_model") realtime_protocol: Final = azure_realtime_protocol_for_client( configured_realtime_protocol, model=model, - base_model=( - configured_base_model.removeprefix(f"{_custom_llm_provider}/") - if isinstance(configured_base_model, str) - else None - ), + base_model=_deployment_base_model(kwargs, _custom_llm_provider), query_params=query_params, realtime_mode=realtime_mode, websocket=websocket, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 44c469f6ed7..071af4135ae 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1091,14 +1091,79 @@ async def test_arealtime_azure_bridges_a_beta_client_to_ga_when_the_deployment_b assert await _azure_session_created_seen_by_a_beta_client(base_model_kwargs) == ( "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", - { - "type": "session.created", - "event_id": "event_1", - "session": {"id": "sess_1", "model": "my-realtime-deployment", "modalities": ["text"], "voice": "marin"}, - }, + _GA_SESSION_CREATED_BRIDGED_TO_BETA, ) +_GA_SESSION_CREATED_BRIDGED_TO_BETA: Final = { + "type": "session.created", + "event_id": "event_1", + "session": {"id": "sess_1", "model": "my-realtime-deployment", "modalities": ["text"], "voice": "marin"}, +} + +_ROUTER_REQUEST_SHAPES: Final = ( + pytest.param({}, id="sdk-router-call"), + pytest.param({"metadata": {"user_api_key_hash": "hashed-key"}}, id="proxy-request-metadata"), +) + + +def _router_with_an_azure_deployment_aliasing(deployment_model: str, base_model: str) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "team-alias", + "litellm_params": { + "model": deployment_model, + "api_base": "https://my-endpoint.openai.azure.com", + "api_key": "fake-key", + }, + "model_info": {"base_model": base_model}, + } + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _ROUTER_REQUEST_SHAPES) +async def test_router_realtime_bridges_a_beta_client_to_ga_when_model_info_base_model_is_ga_only( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, request_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + router: Final = _router_with_an_azure_deployment_aliasing( + "azure/my-realtime-deployment", "azure/gpt-realtime-2.1-mini" + ) + client: Final = _BetaClientThatStaysConnected() + connect: Final = _ConnectToABackendThatSendsOneFrame(_GA_SESSION_CREATED) + + with patch("websockets.connect", connect): + await router._arealtime(model="team-alias", websocket=client, **request_kwargs) + + assert (connect.url, json.loads(client.sent[0])) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _ROUTER_REQUEST_SHAPES) +async def test_router_transcription_session_uses_the_ga_route_when_model_info_base_model_is_ga_only( + local_model_cost_map, request_kwargs: Mapping[str, object] +) -> None: + router: Final = _router_with_an_azure_deployment_aliasing( + "azure/my-whisper-deployment", "azure/gpt-realtime-whisper" + ) + gateway: Final = _GatewayThatRecordsTheUrl() + + await router.acreate_realtime_transcription_session( + model="team-alias", + transcription_session={"input_audio_format": "pcm16"}, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + **request_kwargs, + ) + + assert gateway.url == "https://my-endpoint.openai.azure.com/openai/v1/realtime/transcription_sessions" + + async def _vertex_provider_config_for(monkeypatch, model: str, vertex_location: str | None): from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import VertexChirpRealtimeConfig from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig From 787b620059d2e3e69527a11a8bd9a8330ad834d1 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 21 Oct 2026 21:10:20 -0500 Subject: [PATCH 44/45] fix(realtime): preserve prior merge resolutions after rebase --- litellm/llms/azure/audio_transcriptions.py | 50 ++----- litellm/llms/openai/transcriptions/handler.py | 54 +------- litellm/proxy/_types.py | 4 - litellm/proxy/proxy_server.py | 6 - litellm/proxy/realtime_endpoints/endpoints.py | 17 --- litellm/proxy/utils.py | 11 -- litellm/utils.py | 7 +- .../transcriptions/test_gpt_transcribe.py | 3 + .../proxy/proxy_server/test_routes_audio.py | 5 +- .../test_realtime_webrtc_endpoints.py | 93 +------------ tests/unit/test_cost_calculator.py | 124 ------------------ .../test_model_prices_and_context_window.py | 66 +++++----- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 - 13 files changed, 54 insertions(+), 392 deletions(-) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index d4f53dc0dd5..1d5866bb8fe 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,19 +1,16 @@ from collections.abc import Coroutine, Mapping -from typing import TYPE_CHECKING, Final, cast +from typing import TYPE_CHECKING, Final from openai import AsyncAzureOpenAI, AsyncStream, AzureOpenAI, Stream from openai.types.audio import TranscriptionStreamEvent -from openai.types.audio.transcription_create_params import ( - TranscriptionCreateParamsBase, - TranscriptionCreateParamsStreaming, -) from pydantic import BaseModel from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.llms.base_llm.audio_transcription.transformation import ( + acreate_sdk_transcription, + create_sdk_transcription, sdk_compatible_transcription_request_data, - sdk_compatible_transcription_response, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -98,23 +95,9 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - if data.get("stream") is True: - return azure_client.audio.transcriptions.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsStreaming, sdk_data - ), - timeout=timeout, - ) - - response: Final = sdk_compatible_transcription_response( - azure_client.audio.transcriptions.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsBase, {key: value for key, value in sdk_data.items() if key != "stream"} - ), - stream=False, - timeout=timeout, - ) - ) + _, response = create_sdk_transcription(azure_client, sdk_data, timeout) + if isinstance(response, Stream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() @@ -182,24 +165,9 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - if data.get("stream") is True: - return await async_azure_client.audio.transcriptions.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsStreaming, data - ), - timeout=timeout, - ) - - raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"} - ), - stream=False, - timeout=timeout, - ) - - headers: Final = dict(raw_response.headers) - response = sdk_compatible_transcription_response(raw_response.parse()) + headers, response = await acreate_sdk_transcription(async_azure_client, data, timeout) + if isinstance(response, AsyncStream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 3a55ab01404..23c5e70a3cc 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -4,10 +4,6 @@ from typing import TYPE_CHECKING, Final, Optional, cast import httpx from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream from openai.types.audio import TranscriptionStreamEvent -from openai.types.audio.transcription_create_params import ( - TranscriptionCreateParamsBase, - TranscriptionCreateParamsStreaming, -) from pydantic import BaseModel if TYPE_CHECKING: @@ -17,8 +13,9 @@ from litellm.litellm_core_utils.core_helpers import set_provider_response_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, + acreate_sdk_transcription, + create_sdk_transcription, sdk_compatible_transcription_request_data, - sdk_compatible_transcription_response, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -38,29 +35,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): data: Mapping[str, object], timeout: float | httpx.Timeout, ): - try: - sdk_data: Final = sdk_compatible_transcription_request_data(data) - if data.get("stream") is True: - stream_response: Final = await openai_aclient.audio.transcriptions.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsStreaming, sdk_data - ), - timeout=timeout, - ) - return {}, stream_response - raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsBase, {key: value for key, value in sdk_data.items() if key != "stream"} - ), - stream=False, - timeout=timeout, - ) - headers: Final = dict(raw_response.headers) - parsed_response: Final = sdk_compatible_transcription_response(raw_response.parse()) - - return headers, parsed_response - except Exception as e: - raise e + return await acreate_sdk_transcription(openai_aclient, sdk_compatible_transcription_request_data(data), timeout) def make_sync_openai_audio_transcriptions_request( self, @@ -68,28 +43,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): data: Mapping[str, object], timeout: float | httpx.Timeout, ): - try: - sdk_data: Final = sdk_compatible_transcription_request_data(data) - if data.get("stream") is True: - stream_response: Final = openai_client.audio.transcriptions.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsStreaming, sdk_data - ), - timeout=timeout, - ) - return {}, stream_response - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( - **cast( # cast-ok: SDK file handles have no runtime validation schema - TranscriptionCreateParamsBase, {key: value for key, value in sdk_data.items() if key != "stream"} - ), - stream=False, - timeout=timeout, - ) - headers: Final = dict(raw_response.headers) - response: Final = sdk_compatible_transcription_response(raw_response.parse()) - return headers, response - except Exception as e: - raise e + return create_sdk_transcription(openai_client, sdk_compatible_transcription_request_data(data), timeout) def audio_transcriptions( self, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9f1810226e7..884e68ac1ef 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2490,10 +2490,6 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): admission_queue_timeout_seconds: float = Field( 1.0, gt=0, description="maximum time a request waits for a worker slot" ) - allow_non_billable_realtime_protocols: bool = Field( - False, - description="Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement", - ) plugins: list[PluginConfig] | None = Field( None, description="external services registered as embeddable UI plugins" ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2839026552c..84b7be73542 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13443,12 +13443,6 @@ async def audio_transcriptions( call_type="transcription", ) - if data.get("stream") is True and ProxyLogging.has_post_call_guardrails(data, llm_router): - raise HTTPException( - detail="Streaming transcription does not support output guardrails. Use stream=false.", - status_code=status.HTTP_400_BAD_REQUEST, - ) - ## ROUTE TO CORRECT ENDPOINT ## llm_call: Final = await route_request( data=data, diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index eb337826ece..c23006f13fa 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -47,11 +47,6 @@ _REALTIME_TOKEN_VERSION: Final = "realtime_v1" _DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview" _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" _ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation") -_NON_BILLABLE_REALTIME_PROTOCOL_SETTING: Final = "allow_non_billable_realtime_protocols" -_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE: Final = ( - "Realtime WebRTC endpoints are disabled because provider usage bypasses LiteLLM billing. " - "Set general_settings.allow_non_billable_realtime_protocols to true to opt in" -) @contextmanager @@ -73,15 +68,6 @@ _TRANSLATION_SDP_BODY: Final = { } -def _enforce_non_billable_realtime_protocol_gate(general_settings: Mapping[str, object]) -> None: - if general_settings.get(_NON_BILLABLE_REALTIME_PROTOCOL_SETTING) is True: - return - raise HTTPException( - status_code=http_status.HTTP_403_FORBIDDEN, - detail=_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE, - ) - - def _coerce_realtime_session_type(session_type: str | None) -> str: if session_type in _ALLOWED_SESSION_TYPES: return session_type @@ -352,7 +338,6 @@ async def create_realtime_client_secret( version, ) - _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await read_request_body(request=request) @@ -540,7 +525,6 @@ async def proxy_realtime_calls( media_type="application/json", ) - _enforce_non_billable_realtime_protocol_gate(general_settings) sdp_body: Final[bytes] = await request.body() decoded_payload: Final = _decode_realtime_token_payload(decrypted_token_value) if decoded_payload is not None: @@ -702,7 +686,6 @@ async def create_realtime_transcription_session( version, ) - _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await read_request_body(request=request) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b29bec0bad5..28be11509ae 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3333,17 +3333,6 @@ class ProxyLogging: def has_post_call_response_headers_callbacks() -> bool: return ProxyLogging.callback_capabilities().has_post_call_response_headers - @staticmethod - def has_post_call_guardrails(request_data: Mapping[str, object], llm_router: Router | None) -> bool: - guardrail_data: Final = _check_and_merge_model_level_guardrails( - data=dict(request_data), llm_router=llm_router, trust_client_model_info=False - ) - guardrails, _ = _partition_post_call_callbacks() - return bool(pipeline_managed_guardrail_names(guardrail_data, "post_call")) or any( - guardrail.should_run_guardrail(data=guardrail_data, event_type=GuardrailEventHooks.post_call) - for guardrail in guardrails - ) - @staticmethod def has_streaming_callbacks() -> bool: caps: Final = ProxyLogging.callback_capabilities() diff --git a/litellm/utils.py b/litellm/utils.py index 032192a2504..b58d7d06185 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2047,10 +2047,7 @@ def client(original_function): # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() - if _is_streaming_request( - kwargs=kwargs, - call_type=call_type, - ): + if _returns_requested_stream(kwargs, call_type, result): if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): @@ -2353,7 +2350,7 @@ def client(original_function): raise end_time = datetime.datetime.now() - streaming_requested: Final = _is_streaming_request(kwargs=kwargs, call_type=call_type) + streaming_requested: Final = _returns_requested_stream(kwargs, call_type, result) if streaming_requested or _is_converted_stream_result(result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 613ab467acb..1bf2632a416 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -1,5 +1,7 @@ +import asyncio import io import json +import queue import wave from collections.abc import Callable, Iterator from datetime import datetime @@ -11,6 +13,7 @@ import pytest from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription from litellm.llms.custom_httpx.http_handler import HTTPHandler diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index e0a228b6eea..d22854ed8a0 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -329,12 +329,11 @@ def test_streaming_transcription_keeps_streaming_with_output_guardrails( auth_as: Callable[[], AbstractContextManager[None]], monkeypatch: pytest.MonkeyPatch, configuration: str, - expected_status: int, ) -> None: guardrail: Final = CustomGuardrail( guardrail_name="transcription-output", - event_hook=GuardrailEventHooks.pre_call if configuration == "pre_call" else GuardrailEventHooks.post_call, - default_on=configuration in ("default", "pre_call"), + event_hook=GuardrailEventHooks.post_call, + default_on=configuration == "default", ) monkeypatch.setattr(litellm, "callbacks", [guardrail]) if configuration == "policy": diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 2e3e3d4f7d7..416fe132742 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -20,7 +20,7 @@ from starlette.routing import Match import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot -from litellm.proxy._types import ConfigGeneralSettings, UserAPIKeyAuth +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, @@ -131,100 +131,9 @@ def proxy_app(monkeypatch): from litellm.proxy import proxy_server monkeypatch.setattr(proxy_server, "master_key", "sk-test-master-key") - monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", True) return proxy_server.app -def test_non_billable_realtime_protocols_default_to_disabled(): - assert ConfigGeneralSettings().allow_non_billable_realtime_protocols is False - - -@pytest.mark.parametrize( - ("path", "body"), - ( - ( - "/v1/realtime/client_secrets", - {"model": "gpt-realtime-2"}, - ), - ( - "/v1/realtime/translations/client_secrets", - { - "model": "gpt-realtime-translate", - "session": {"type": "translation", "model": "gpt-realtime-translate"}, - }, - ), - ( - "/v1/realtime/transcription_sessions", - {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, - ), - ), -) -def test_non_billable_realtime_credential_endpoints_require_opt_in( - proxy_app, - monkeypatch, - path, - body, -): - from litellm.proxy import proxy_server - - monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") - try: - client = TestClient(proxy_app, raise_server_exceptions=False) - with patch( # test-quality-ok: endpoint gate must prove routing is never reached - "litellm.proxy.proxy_server.route_request" - ) as mock_route_request: - response = client.post( - path, - headers={"Authorization": "Bearer sk-test-master-key"}, - json=body, - ) - - assert response.status_code == 403 - assert "bypasses LiteLLM billing" in response.json()["detail"] - mock_route_request.assert_not_called() - finally: - proxy_app.dependency_overrides.pop(user_api_key_auth, None) - - -@pytest.mark.parametrize( - "path", - ( - "/v1/realtime/calls", - "/v1/realtime/translations/calls", - ), -) -def test_non_billable_realtime_sdp_endpoints_require_opt_in( - proxy_app, - monkeypatch, - path, -): - from litellm.proxy import proxy_server - - monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) - token_payload = _encode_realtime_token_payload( - ephemeral_key="epk", - model_id="gpt-realtime-2", - user_id=None, - team_id=None, - expires_at=int(time.time()) + 3600, - ) - encrypted_token = encrypt_value_helper(token_payload) - client = TestClient(proxy_app, raise_server_exceptions=False) - with patch( # test-quality-ok: endpoint gate must prove routing is never reached - "litellm.proxy.proxy_server.route_request" - ) as mock_route_request: - response = client.post( - path, - headers={"Authorization": f"Bearer {encrypted_token}"}, - content=b"v=0\r\n", - ) - - assert response.status_code == 403 - assert "bypasses LiteLLM billing" in response.json()["detail"] - mock_route_request.assert_not_called() - - @pytest.fixture def mock_route_request_client_secrets(): """Mock route_request to return a fake upstream client_secrets response.""" diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 84763b58351..818719b50bb 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -10677,127 +10677,3 @@ def test_bedrock_registered_cache_prices_preserve_dynamic_threshold_rates( assert entry is not None assert entry[1]["input_cost_per_token_above_300k_tokens"] == 0.03 assert entry[1][tier_field] == (tier_override if tier_override is not None else 0.007) - - -@pytest.mark.parametrize("provider", ("openai", "azure")) -@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) -def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str): - model_name: Final = f"azure/{model}" if provider == "azure" else model - rates: Final = litellm.model_cost[model_name] - events: Final[OpenAIRealtimeStreamList] = [ - {"type": "session.created", "session": {"model": model}}, - { - "type": "response.done", - "response": { - "usage": { - "input_tokens": 1000, - "output_tokens": 300, - "total_tokens": 1300, - "input_token_details": { - "text_tokens": 400, - "audio_tokens": 400, - "image_tokens": 200, - "cached_tokens": 300, - "cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50}, - }, - "output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100}, - } - }, - }, - ] - combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events) - actual: Final = handle_realtime_stream_cost_calculation( - results=events, - combined_usage_object=combined, - custom_llm_provider=provider, - litellm_model_name=model_name, - ) - expected: Final = ( - 300 * rates["input_cost_per_token"] - + 250 * rates["input_cost_per_audio_token"] - + 150 * rates["input_cost_per_image_token"] - + 100 * rates["cache_read_input_token_cost"] - + 150 * rates["cache_read_input_audio_token_cost"] - + 50 * rates["cache_read_input_image_token_cost"] - + 200 * rates["output_cost_per_token"] - + 100 * rates["output_cost_per_audio_token"] - ) - - assert actual == pytest.approx(expected) - - -def test_realtime_translation_duration_cost(_local_model_cost_map): - from litellm.cost_calculator import handle_realtime_translation_cost_calculation - - model: Final = "gpt-realtime-translate" - events: Final[OpenAIRealtimeStreamList] = [ - {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}} - ] - actual: Final = handle_realtime_translation_cost_calculation( - results=events, - custom_llm_provider="openai", - litellm_model_name=model, - ) - - assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) - - -@pytest.mark.parametrize("output_seconds", [None, 2.0]) -def test_realtime_translation_duration_cost_includes_provider_input_usage( - _local_model_cost_map, output_seconds: float | None -): - from litellm.cost_calculator import handle_realtime_translation_cost_calculation - - model: Final = "gpt-realtime-translate" - events: Final[OpenAIRealtimeStreamList] = [ - { - "type": "session.closed", - "usage": { - "type": "duration", - "input_seconds": 3.0, - **({"output_seconds": output_seconds} if output_seconds is not None else {}), - }, - } - ] - actual: Final = handle_realtime_translation_cost_calculation( - results=events, - custom_llm_provider="openai", - litellm_model_name=model, - ) - - expected: Final = ( - 3 * litellm.model_cost[model]["input_cost_per_second"] - + (output_seconds or 0) * litellm.model_cost[model]["output_cost_per_second"] - ) - assert actual == pytest.approx(expected) - - -@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True]) -def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( - _local_model_cost_map, malformed_duration: float | int | bool -) -> None: - from litellm.cost_calculator import handle_realtime_translation_cost_calculation - - model: Final = "gpt-realtime-translate" - input_invalid_events: Final[OpenAIRealtimeStreamList] = [ - { - "type": "session.closed", - "usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5}, - } - ] - output_invalid_events: Final[OpenAIRealtimeStreamList] = [ - { - "type": "session.closed", - "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration}, - } - ] - - input_invalid_cost: Final = handle_realtime_translation_cost_calculation( - results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model - ) - output_invalid_cost: Final = handle_realtime_translation_cost_calculation( - results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model - ) - - assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"]) - assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 73a9af91eef..ffbc3d2d784 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -743,6 +743,39 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): ] +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["cache_read_input_image_token_cost"] > 0 + assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"] + assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"] + + +@pytest.mark.parametrize( + "model,mode", + ( + ("gpt-realtime-translate", "realtime"), + ("gpt-live-transcribe", "audio_transcription"), + ("gpt-transcribe", "audio_transcription"), + ), +) +def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["mode"] == azure["mode"] == mode + assert azure["supports_audio_input"] is True + assert azure["supported_endpoints"] + + +def test_model_prices_backup_is_synchronized(prices: dict): + backup: Final = json.loads(BACKUP_PRICES_PATH.read_text()) + + assert backup == prices + + def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]: prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) return MappingProxyType( @@ -791,36 +824,3 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa sibling = rows["volcengine/doubao-seed-2-1-pro-260628"] assert isinstance(sibling, dict) assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")) - - -@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) -def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str): - openai: Final = prices[model] - azure: Final = prices[f"azure/{model}"] - - assert openai["cache_read_input_image_token_cost"] > 0 - assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"] - assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"] - - -@pytest.mark.parametrize( - "model,mode", - ( - ("gpt-realtime-translate", "realtime"), - ("gpt-live-transcribe", "audio_transcription"), - ("gpt-transcribe", "audio_transcription"), - ), -) -def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str): - openai: Final = prices[model] - azure: Final = prices[f"azure/{model}"] - - assert openai["mode"] == azure["mode"] == mode - assert azure["supports_audio_input"] is True - assert azure["supported_endpoints"] - - -def test_model_prices_backup_is_synchronized(prices: dict): - backup: Final = json.loads(BACKUP_PRICES_PATH.read_text()) - - assert backup == prices diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bc6fab08c5e..e7d85c4a1e0 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29943,12 +29943,6 @@ export interface components { * @description opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine */ allow_cli_sso_verification_uri_complete?: boolean | null; - /** - * Allow Non Billable Realtime Protocols - * @description Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement - * @default false - */ - allow_non_billable_realtime_protocols: boolean; /** * Allow Unmanaged Response Ids * @description If True, lets keys address Responses API ids that this proxy did not issue (raw provider ids, or ids issued before response-id encryption was configured). Such an id carries no owner, so no ownership check can run on it; ids this proxy did issue keep full ownership enforcement. Off by default, in which case an unrecognized response id is rejected with 403 From a4bcd7cf6b90776986e468110cc1df51739bdcdd Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 21 Oct 2026 21:19:34 -0500 Subject: [PATCH 45/45] fix(realtime): enforce translation policies and normalize Azure transcription URLs --- .../litellm_core_utils/realtime_streaming.py | 20 ++- .../azure/realtime/http_transformation.py | 2 +- litellm/realtime_api/main.py | 4 +- .../test_realtime_streaming.py | 2 +- .../realtime/test_http_transformation.py | 23 ++- tests/unit/realtime_api/test_main.py | 141 ++++++++++++++---- 6 files changed, 160 insertions(+), 32 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 66195634efc..d391f17eba4 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -327,6 +327,24 @@ def _realtime_runner_guardrail( return callback if runs else None +def translation_guardrails_apply(request_data: Mapping[str, object]) -> bool: + from litellm.types.guardrails import GuardrailEventHooks + + if _guardrails_run_for_event_hooks(request_data, (GuardrailEventHooks.post_call,)): + return True + if any( + isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket + for bucket in (request_data.get("metadata"), request_data.get("litellm_metadata")) + ): + from litellm.proxy.utils import pipeline_managed_guardrail_names + + if pipeline_managed_guardrail_names(request_data, "pre_call") or pipeline_managed_guardrail_names( + request_data, "post_call" + ): + return True + return realtime_guardrails_apply(request_data) + + def realtime_guardrails_apply(request_data: Mapping[str, object]) -> bool: from litellm.types.guardrails import GuardrailEventHooks @@ -2039,7 +2057,7 @@ class RealTimeStreaming: return ClientLoopExit.CLIENT_DISCONNECTED async def bidirectional_forward(self) -> None: - if self._is_translation_session and self._has_realtime_guardrails(): + if self._is_translation_session and translation_guardrails_apply(self.request_data): await self.websocket.send_text( realtime_error_event(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, error_type="guardrail_error") ) diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 732b0946c6d..a8136a8fe89 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -66,7 +66,7 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base) if self._uses_ga_api(model, api_version): - return f"{base.rstrip('/')}/openai/v1/realtime/transcription_sessions" + return _azure_realtime_v1_url(base, "transcription_sessions") version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}" diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index b7f99992593..ba41fa58635 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -48,7 +48,7 @@ from ..litellm_core_utils.realtime_errors import realtime_error_event, websocket from ..litellm_core_utils.realtime_streaming import ( TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, client_websocket_scope, - realtime_guardrails_apply, + translation_guardrails_apply, ) from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client @@ -164,7 +164,7 @@ def _translation_rejection( ) -> _TranslationRejection | None: if custom_llm_provider not in supported_providers: return _TranslationProviderUnsupported(provider=custom_llm_provider, supported_providers=supported_providers) - if realtime_guardrails_apply({"litellm_metadata": dict(litellm_metadata)}): + if translation_guardrails_apply({"litellm_metadata": dict(litellm_metadata)}): return _TranslationGuardrailsUnsupported() return None diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 729e5bb1576..a0f89772ca3 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3397,7 +3397,7 @@ async def test_translation_rejects_applicable_guardrails_before_forwarding_audio await streaming.bidirectional_forward() messages: Final = tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) - if (default_on or requested) and event_hook is not GuardrailEventHooks.post_call: + if default_on or requested: assert messages[0]["error"]["type"] == "guardrail_error" client.close.assert_awaited_once_with( code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" diff --git a/tests/unit/llms/azure/realtime/test_http_transformation.py b/tests/unit/llms/azure/realtime/test_http_transformation.py index 714ce9558cf..7c613b4c9a8 100644 --- a/tests/unit/llms/azure/realtime/test_http_transformation.py +++ b/tests/unit/llms/azure/realtime/test_http_transformation.py @@ -7,7 +7,11 @@ import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler _DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview" -_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai") +_API_BASES: Final = ( + "https://my.openai.azure.com", + "https://my.openai.azure.com/", + "https://my.openai.azure.com/openai", +) class _AzureThatRecordsTheRequest: @@ -55,3 +59,20 @@ async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_k assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls" assert request.headers.get("authorization") == "Bearer ek_upstream" assert "api-key" not in request.headers + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_base", (*_API_BASES, "https://my.openai.azure.com/openai/v1/")) +async def test_azure_ga_transcription_session_normalizes_the_api_base(api_base: str) -> None: + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_transcription_session( + model="azure/transcription-deployment", + transcription_session={"input_audio_transcription": {"model": "transcription-deployment"}}, + api_base=api_base, + api_key="azure-key", + api_version="v1", + client=azure.client(), + ) + (request,) = azure.requests + assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/transcription_sessions" + assert request.headers.get("api-key") == "azure-key" diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 071af4135ae..197c1e86917 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -10,7 +10,7 @@ from datetime import datetime, timedelta, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from types import TracebackType -from typing import ClassVar, Final +from typing import ClassVar, Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -37,6 +37,9 @@ from litellm.realtime_api import main as realtime_main from litellm.realtime_api.main import with_resolved_session_model from litellm.types.guardrails import GuardrailEventHooks from litellm.types.realtime import RealtimeQueryParams +from litellm.proxy.litellm_pre_call_utils import _apply_resolved_guardrails_to_metadata +from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyMatchContext +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep @pytest.fixture @@ -600,7 +603,11 @@ async def test_bearer_health_check_sends_the_secret_behind_an_oidc_credential( monkeypatch.setattr( litellm, "credential_list", - [CredentialItem(credential_name="voice-sa", credential_values={"api_key": f"oidc/file/{token}"}, credential_info={})], + [ + CredentialItem( + credential_name="voice-sa", credential_values={"api_key": f"oidc/file/{token}"}, credential_info={} + ) + ], ) connect: Final = _CapturingConnect() with patch("websockets.connect", connect): @@ -1597,10 +1604,13 @@ async def test_openai_realtime_http_routes_send_the_federated_bearer_for_a_keyle @pytest.mark.asyncio @pytest.mark.parametrize("flow", ("client_secret", "calls")) async def test_translation_http_routes_preserve_explicit_azure_provider(flow: str) -> None: - expected_path: Final = "/openai/v1/realtime/translations/" + { - "client_secret": "client_secrets", - "calls": "calls", - }[flow] + expected_path: Final = ( + "/openai/v1/realtime/translations/" + + { + "client_secret": "client_secrets", + "calls": "calls", + }[flow] + ) def send_response(request: httpx.Request) -> httpx.Response: assert request.url.host == "azure-deployment.example" @@ -1923,24 +1933,25 @@ class _RecordingClientWebSocket: self.close_code = code -def _block_every_request_with_a_guardrail(monkeypatch: pytest.MonkeyPatch) -> None: +def _block_every_request_with_a_guardrail( + monkeypatch: pytest.MonkeyPatch, event_hook: GuardrailEventHooks = GuardrailEventHooks.pre_call +) -> None: monkeypatch.setattr( litellm, "callbacks", - [ - CustomGuardrail( - guardrail_name="translation-blocker", event_hook=GuardrailEventHooks.pre_call, default_on=True - ) - ], + [CustomGuardrail(guardrail_name="translation-blocker", event_hook=event_hook, default_on=True)], ) @pytest.mark.asyncio @pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) async def test_translation_http_routes_refuse_a_guarded_request_before_any_upstream_call( - route: str, monkeypatch: pytest.MonkeyPatch + route: str, + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, ) -> None: - _block_every_request_with_a_guardrail(monkeypatch) + _block_every_request_with_a_guardrail(monkeypatch, event_hook) upstream: Final = _UpstreamThatMustNotBeCalled() with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) as raised: @@ -1970,10 +1981,15 @@ _DEFAULT_ON_GUARDRAIL_OPT_OUTS: Final = ( (("translation_client_secrets", "client_secrets"), ("translation_calls", "calls")), ) @pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) async def test_translation_http_routes_serve_a_key_opted_out_of_the_default_on_guardrail( - route: str, path: str, key_metadata: Mapping[str, object], monkeypatch: pytest.MonkeyPatch + route: str, + path: str, + key_metadata: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, ) -> None: - _block_every_request_with_a_guardrail(monkeypatch) + _block_every_request_with_a_guardrail(monkeypatch, event_hook) gateway: Final = _GatewayThatRecordsTheUrl() await _realtime_http_route_call( @@ -2009,10 +2025,12 @@ async def test_translation_http_routes_refuse_a_provider_other_than_openai_and_a @pytest.mark.asyncio +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) async def test_translation_websocket_closes_1008_for_a_guarded_request_before_the_upstream_connect( monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, ) -> None: - _block_every_request_with_a_guardrail(monkeypatch) + _block_every_request_with_a_guardrail(monkeypatch, event_hook) websocket: Final = _RecordingClientWebSocket() connect: Final = _CapturingConnect() @@ -2034,10 +2052,13 @@ async def test_translation_websocket_closes_1008_for_a_guarded_request_before_th @pytest.mark.asyncio @pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) async def test_translation_websocket_opens_for_a_key_opted_out_of_the_default_on_guardrail( - key_metadata: Mapping[str, object], monkeypatch: pytest.MonkeyPatch + key_metadata: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, ) -> None: - _block_every_request_with_a_guardrail(monkeypatch) + _block_every_request_with_a_guardrail(monkeypatch, event_hook) websocket: Final = _RecordingClientWebSocket() connect: Final = _CapturingConnect() @@ -2093,17 +2114,17 @@ class _NativeLifecycleGuardrail(CustomGuardrail): @pytest.mark.asyncio -@pytest.mark.parametrize( - ("guardrail_class", "event_hook"), - ((_NativeLifecycleGuardrail, GuardrailEventHooks.pre_call), (CustomGuardrail, GuardrailEventHooks.post_call)), -) -async def test_translation_websocket_opens_under_a_default_on_guardrail_the_realtime_runner_skips( - guardrail_class: type[CustomGuardrail], event_hook: GuardrailEventHooks, monkeypatch: pytest.MonkeyPatch +async def test_translation_websocket_opens_under_a_native_pre_call_guardrail( + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( litellm, "callbacks", - [guardrail_class(guardrail_name="runner-skipped", event_hook=event_hook, default_on=True)], + [ + _NativeLifecycleGuardrail( + guardrail_name="runner-skipped", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + ], ) websocket: Final = _RecordingClientWebSocket() connect: Final = _CapturingConnect() @@ -2140,3 +2161,71 @@ async def test_translation_websocket_closes_1008_for_a_provider_other_than_opena assert [json.loads(event)["error"]["message"] for event in websocket.sent] == [ "Realtime translation supports only the azure, azure_ai, openai providers, got xai" ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls", "websocket")) +@pytest.mark.parametrize("mode", ("pre_call", "post_call")) +@pytest.mark.parametrize("metadata_bucket", ("metadata", "litellm_metadata")) +@pytest.mark.parametrize("selected,condition_matches", ((True, True), (True, False), (False, True))) +async def test_translation_rejects_only_applicable_policy_pipelines( + route: str, + mode: Literal["pre_call", "post_call"], + metadata_bucket: str, + selected: bool, + condition_matches: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + request_data: Final[dict[str, object]] = {"metadata": {}, "litellm_metadata": {}} + _apply_resolved_guardrails_to_metadata( + data=request_data, + metadata_variable_name=metadata_bucket, + context=PolicyMatchContext(model="translation-deployment"), + policy_names=["translation-policy"] if selected else [], + policies={ + "translation-policy": Policy( + condition=PolicyCondition(model="translation-deployment" if condition_matches else "unrelated-model"), + pipeline=GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="speech-policy", on_fail="block")]), + ) + }, + ) + metadata: Final = realtime_main._build_litellm_metadata(request_data) + blocked: Final = selected and condition_matches + if route == "websocket": + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + litellm_metadata=metadata, + ) + if blocked: + assert connect.url is None + assert websocket.close_code == 1008 + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in websocket.sent[0] + else: + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + return + upstream: Final = _UpstreamThatMustNotBeCalled() + call: Final = _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + metadata=metadata, + ) + if blocked: + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) as raised: + await call + assert raised.value.status_code == 400 + assert upstream.called is False + else: + response: Final = await call + assert response.status_code == 200 + assert upstream.called is True