From bd43c233ef9a4b402793c912e85b543119f0dcb7 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 12:12:59 -0500 Subject: [PATCH 01/21] feat(realtime): support latest OpenAI audio models --- cookbook/gpt_realtime_translate.py | 242 ++++++++++ litellm/__init__.py | 18 +- litellm/constants.py | 34 ++ litellm/cost_calculator.py | 76 +++- .../audio_utils/transcription_streaming.py | 204 +++++++++ .../litellm_core_utils/llm_cost_calc/utils.py | 79 +++- .../convert_dict_to_response.py | 15 +- .../litellm_core_utils/realtime_streaming.py | 237 ++++++++-- litellm/llms/azure/audio_transcriptions.py | 21 +- litellm/llms/azure/realtime/handler.py | 34 +- .../azure/realtime/http_transformation.py | 27 +- .../base_llm/realtime/http_transformation.py | 10 + litellm/llms/custom_httpx/llm_http_handler.py | 287 ++++++++++-- litellm/llms/openai/realtime/handler.py | 126 +++++- .../openai/realtime/http_transformation.py | 12 + .../transcriptions/gpt_transformation.py | 30 +- litellm/llms/openai/transcriptions/handler.py | 42 +- litellm/main.py | 95 +++- litellm/proxy/_lazy_features.py | 15 + litellm/proxy/_lazy_openapi_snapshot.json | 180 ++++++++ litellm/proxy/_types.py | 13 + litellm/proxy/common_request_processing.py | 2 + .../proxy/common_utils/http_parsing_utils.py | 20 +- litellm/proxy/proxy_server.py | 88 +++- litellm/proxy/realtime_endpoints/endpoints.py | 138 +++++- litellm/proxy/route_llm_request.py | 2 + litellm/realtime_api/README.md | 14 +- litellm/realtime_api/main.py | 291 ++++++++++-- litellm/responses/streaming_iterator.py | 2 + litellm/responses/utils.py | 8 +- litellm/router.py | 12 + litellm/types/llms/openai.py | 14 + litellm/types/realtime.py | 48 +- litellm/types/utils.py | 38 +- litellm/utils.py | 33 +- .../llm_nonconversational.yaml | 6 + tests/e2e/coverage_registry/schema.py | 11 +- .../test_realtime_streaming.py | 187 ++++++++ .../llms/azure/test_azure_common_utils.py | 5 + .../realtime/test_openai_realtime_handler.py | 324 +++++++------- .../realtime/test_transcription_sessions.py | 55 ++- .../llms/openai/realtime/test_translation.py | 265 +++++++++++ .../proxy/auth/test_route_checks.py | 31 ++ .../proxy/proxy_server/test_routes_audio.py | 57 ++- .../test_realtime_webrtc_endpoints.py | 414 +++++++++++++++--- tests/test_litellm/proxy/test_proxy_server.py | 14 +- .../test_streaming_iterator_error_events.py | 2 + tests/test_litellm/test_utils.py | 3 + .../transcriptions/test_gpt_transcribe.py | 297 +++++++++++++ .../test_transcription_duration_hidden.py | 62 ++- 50 files changed, 3744 insertions(+), 496 deletions(-) create mode 100644 cookbook/gpt_realtime_translate.py create mode 100644 litellm/litellm_core_utils/audio_utils/transcription_streaming.py create mode 100644 tests/test_litellm/llms/openai/realtime/test_translation.py create mode 100644 tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py new file mode 100644 index 00000000000..73c91eaa1e9 --- /dev/null +++ b/cookbook/gpt_realtime_translate.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import base64 +import json +import os +import sys +import wave +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode, urlsplit, urlunsplit + +import websockets +from websockets.asyncio.client import ClientConnection + +SAMPLE_RATE = 24_000 +CHANNELS = 1 +SAMPLE_WIDTH = 2 +CHUNK_DURATION_SECONDS = 0.1 +CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS) +OUTPUT_IDLE_TIMEOUT_SECONDS = 3.0 +INITIAL_RESPONSE_TIMEOUT_SECONDS = 30.0 +AUDIO_EVENT_TYPES = frozenset( + { + "session.output_audio.delta", + "response.audio.delta", + "response.output_audio.delta", + } +) +TRANSCRIPT_EVENT_TYPES = frozenset( + { + "session.output_transcript.delta", + "response.text.delta", + "response.output_audio_transcript.delta", + } +) + + +@dataclass(frozen=True, slots=True) +class Settings: + input_wav: Path + output_wav: Path + base_url: str + model: str + target_language: str + trailing_silence_seconds: float + api_key: str + + +def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None: + sys.stdout.write(f"{message}{end}") + if flush: + sys.stdout.flush() + + +def write_stderr(message: str) -> None: + sys.stderr.write(f"{message}\n") + + +def parse_args(argv: Sequence[str] | None = None) -> Settings | str: + parser = argparse.ArgumentParser( + description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio", + ) + parser.add_argument("input_wav", type=Path) + parser.add_argument("--output", type=Path, default=Path("translated.wav")) + parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000")) + parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate")) + parser.add_argument("--target-language", default="fr") + parser.add_argument("--trailing-silence", type=float, default=1.5) + parsed = parser.parse_args(argv) + api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY") + if not api_key: + return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script" + if parsed.trailing_silence < 0: + return "--trailing-silence must be zero or greater" + return Settings( + input_wav=parsed.input_wav, + output_wav=parsed.output, + base_url=parsed.base_url, + model=parsed.model, + target_language=parsed.target_language, + trailing_silence_seconds=parsed.trailing_silence, + api_key=api_key, + ) + + +def translation_url(base_url: str, model: str) -> str | None: + parsed = urlsplit(base_url.rstrip("/")) + scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme) + if not scheme or not parsed.netloc: + return None + base_path = parsed.path.rstrip("/") + realtime_path = ( + f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations" + ) + return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), "")) + + +def read_pcm16_wav(path: Path) -> bytes | str: + try: + with wave.open(str(path), "rb") as source: + actual_format = ( + source.getnchannels(), + source.getsampwidth(), + source.getframerate(), + source.getcomptype(), + ) + expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE") + if actual_format != expected_format: + return ( + f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received " + f"channels={actual_format[0]}, sample_width={actual_format[1]}, " + f"sample_rate={actual_format[2]}, compression={actual_format[3]}" + ) + return source.readframes(source.getnframes()) + except (OSError, EOFError, wave.Error) as exc: + return f"Unable to read {path}: {exc}" + + +def audio_chunks(audio: bytes) -> Iterator[bytes]: + return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES)) + + +def audio_message(audio: bytes) -> str: + return json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(audio).decode("ascii"), + } + ) + + +async def configure_session(connection: ClientConnection, target_language: str) -> str | None: + await connection.send( + json.dumps( + { + "type": "session.update", + "session": {"audio": {"output": {"language": target_language}}}, + } + ) + ) + while True: + raw_event = await asyncio.wait_for(connection.recv(), timeout=20) + event = json.loads(raw_event) + event_type = event.get("type") + if event_type == "session.created": + write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}") + if event_type == "session.updated": + return None + if event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + + +async def send_audio( + connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event +) -> None: + silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + try: + for chunk in audio_chunks(pcm + silence): + await connection.send(audio_message(chunk)) + await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + finally: + finished.set() + + +async def receive_translation( + connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event +) -> str | None: + audio_received = asyncio.Event() + try: + with wave.open(str(output_path), "wb") as output: + output.setnchannels(CHANNELS) + output.setsampwidth(SAMPLE_WIDTH) + output.setframerate(SAMPLE_RATE) + write_stdout("Translation: ", end="", flush=True) + while True: + timeout = OUTPUT_IDLE_TIMEOUT_SECONDS if sender_finished.is_set() else INITIAL_RESPONSE_TIMEOUT_SECONDS + try: + raw_event = await asyncio.wait_for(connection.recv(), timeout=timeout) + except TimeoutError: + if sender_finished.is_set() and audio_received.is_set(): + write_stdout() + return None + return "The translation stream ended without translated audio" + event = json.loads(raw_event) + event_type = event.get("type") + if event_type in AUDIO_EVENT_TYPES: + output.writeframes(base64.b64decode(event.get("delta", ""), validate=True)) + audio_received.set() + elif event_type in TRANSCRIPT_EVENT_TYPES: + write_stdout(event.get("delta", event.get("text", "")), end="", flush=True) + elif event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + except (OSError, wave.Error) as exc: + return f"Unable to write {output_path}: {exc}" + + +async def translate(settings: Settings, pcm: bytes) -> str | None: + url = translation_url(settings.base_url, settings.model) + if not url: + return f"Invalid --base-url: {settings.base_url}" + sender_finished = asyncio.Event() + try: + async with websockets.connect( + url, + additional_headers={"Authorization": f"Bearer {settings.api_key}"}, + proxy=None, + open_timeout=20, + close_timeout=5, + ) as connection: + configuration_error = await configure_session(connection, settings.target_language) + if configuration_error: + return configuration_error + async with asyncio.TaskGroup() as tasks: + receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished)) + tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished)) + return receiver.result() + except Exception as exc: + return f"Translation failed: {type(exc).__name__}: {exc}" + + +def main(argv: Sequence[str] | None = None) -> int: + settings = parse_args(argv) + if isinstance(settings, str): + write_stderr(settings) + return 2 + pcm = read_pcm16_wav(settings.input_wav) + if isinstance(pcm, str): + write_stderr(pcm) + return 2 + error = asyncio.run(translate(settings, pcm)) + if error: + write_stderr(error) + return 1 + write_stdout(f"Translated audio: {settings.output_wav}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/litellm/__init__.py b/litellm/__init__.py index c8df4394a06..4ec7aeea8fb 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -29,7 +29,7 @@ def _dev_env_hot_reload_enabled() -> bool: if os.getenv("LITELLM_MODE", "DEV") == "DEV": _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled()) -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from typing import ( Any, Callable, @@ -72,6 +72,7 @@ from litellm.constants import ( OPENAI_CHAT_COMPLETION_PARAMS as _openai_completion_params, # backwards compatibility OPENAI_FINISH_REASONS, OPENAI_FINISH_REASONS as _openai_finish_reasons, # backwards compatibility + OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS, openai_compatible_endpoints, openai_compatible_providers, openai_text_completion_compatible_providers, @@ -1007,6 +1008,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None): _populate_provider_model_sets(model_cost) +open_ai_chat_completion_models.update(OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS) # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary # this is maintained for Exception Mapping @@ -1466,7 +1468,9 @@ from .realtime_api.main import ( _arealtime, acreate_realtime_client_secret, acreate_realtime_transcription_session, + acreate_realtime_translation_client_secret, arealtime_calls, + arealtime_translation_calls, ) from .responses.main import _aresponses_websocket from .fine_tuning.main import * @@ -1650,6 +1654,9 @@ if TYPE_CHECKING: from .llms.vertex_ai.rerank.transformation import ( VertexAIRerankConfig as VertexAIRerankConfig, ) + from .llms.together_ai.chat.transformation import ( + TogetherAIChatConfig as TogetherAIChatConfig, + ) from .llms.fireworks_ai.rerank.transformation import ( FireworksAIRerankConfig as FireworksAIRerankConfig, ) @@ -1695,9 +1702,6 @@ if TYPE_CHECKING: BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig, ) from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig - from .llms.together_ai.chat.transformation import ( - TogetherAIChatConfig as TogetherAIChatConfig, - ) from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig as VertexGeminiConfig, @@ -1853,6 +1857,9 @@ if TYPE_CHECKING: from .llms.xai.responses.transformation import ( XAIResponsesAPIConfig as XAIResponsesAPIConfig, ) + from .llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig as VertexAIInteractionsConfig, + ) from .llms.litellm_proxy.responses.transformation import ( LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, ) @@ -1877,9 +1884,6 @@ if TYPE_CHECKING: from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) - from .llms.vertex_ai.interactions.transformation import ( - VertexAIInteractionsConfig as VertexAIInteractionsConfig, - ) from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config, diff --git a/litellm/constants.py b/litellm/constants.py index e5b662bd515..855f51811a4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -828,10 +828,44 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "language", + "languages", + "keywords", "response_format", + "stream", "timestamp_granularities", ] +OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS: Final = frozenset( + { + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-translate", + "gpt-realtime-whisper", + "gpt-transcribe", + "gpt-live-transcribe", + } +) + +AZURE_GA_REALTIME_MODELS: Final = frozenset( + { + "gpt-realtime-2", + "gpt-realtime-2-2026-05-06", + "gpt-realtime-2.1", + "gpt-realtime-2.1-2026-07-07", + "gpt-realtime-2.1-mini", + "gpt-realtime-2.1-mini-2026-07-07", + "gpt-realtime-translate", + "gpt-realtime-translate-2026-05-06", + "gpt-realtime-translate-2026-05-07", + "gpt-realtime-whisper", + "gpt-realtime-whisper-2026-05-06", + "gpt-realtime-whisper-2026-05-07", + "gpt-transcribe", + "gpt-live-transcribe", + } +) + OPENAI_EMBEDDING_PARAMS: Final = ["dimensions", "encoding_format", "user"] DEFAULT_EMBEDDING_PARAM_VALUES: Final = { diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7990832dc48..2e4448475a7 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1006,6 +1006,25 @@ def get_usage_object( return None +def _get_transcription_usage_duration(completion_response: object) -> float | None: + usage_object: Final = ( + completion_response.get("usage") + if isinstance(completion_response, dict) + else getattr(completion_response, "usage", None) + ) + usage_type: Final = ( + usage_object.get("type") if isinstance(usage_object, dict) else getattr(usage_object, "type", None) + ) + if usage_type != "duration": + return None + seconds: Final = ( + usage_object.get("seconds") if isinstance(usage_object, dict) else getattr(usage_object, "seconds", None) + ) + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds < 0: + return None + return float(seconds) + + def _is_known_usage_objects(usage_obj): """Returns True if the usage obj is a known Usage type""" return ( @@ -1595,9 +1614,14 @@ def completion_cost( # the response attribute (for verbose_json responses that # naturally include duration from the provider). _hidden = getattr(completion_response, "_hidden_params", {}) or {} - audio_transcription_file_duration = _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), + provider_duration = _get_transcription_usage_duration(completion_response) + audio_transcription_file_duration = ( + provider_duration + if provider_duration is not None + else _hidden.get( + "audio_transcription_duration", + getattr(completion_response, "duration", 0.0), + ) ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): @@ -2848,6 +2872,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor): _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" +_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed" def _candidate_realtime_token_costs( @@ -2947,7 +2972,21 @@ def handle_realtime_stream_cost_calculation( if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results) else 0.0 ) - total_cost: Final = input_cost_per_token + output_cost_per_token + transcription_cost + translation_cost: Final = handle_realtime_translation_cost_calculation( + results=results, + custom_llm_provider=custom_llm_provider, + litellm_model_name=litellm_model_name, + ) + total_cost: Final = input_cost_per_token + output_cost_per_token + transcription_cost + translation_cost + + additional_costs: Final = { # mutable-ok: logging stores a mutable per-request cost breakdown + key: value + for key, value in ( + ("transcription_cost", transcription_cost), + ("translation_cost", translation_cost), + ) + if value > 0 + } _store_cost_breakdown_in_logging_obj( litellm_logging_obj=litellm_logging_obj, @@ -2955,13 +2994,40 @@ def handle_realtime_stream_cost_calculation( completion_tokens_cost_usd_dollar=output_cost_per_token, cost_for_built_in_tools_cost_usd_dollar=0.0, total_cost_usd_dollar=total_cost, - additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None, data_residency=data_residency, + additional_costs=additional_costs or None, ) return total_cost +def handle_realtime_translation_cost_calculation( + results: OpenAIRealtimeStreamList, + custom_llm_provider: str, + litellm_model_name: str, +) -> float: + output_seconds = 0.0 # rebind-ok: duration is accumulated across translation close events + for result in results: + if result.get("type") != _TRANSLATION_CLOSED_EVENT_TYPE: + continue + usage = result.get("usage") + if isinstance(usage, dict) and isinstance(usage.get("output_seconds"), (int, float)): + output_seconds += float(usage["output_seconds"]) + if output_seconds <= 0: + return 0.0 + try: + model_info: Final = litellm.get_model_info( + model=litellm_model_name, + custom_llm_provider=custom_llm_provider, + ) + except Exception: # noqa: BLE001 # unknown model metadata should yield zero translation cost + return 0.0 + output_cost_per_second: Final = model_info.get("output_cost_per_second") + if not isinstance(output_cost_per_second, (int, float)): + return 0.0 + return output_seconds * output_cost_per_second + + def handle_realtime_transcription_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py new file mode 100644 index 00000000000..5ae246f3468 --- /dev/null +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import datetime +import traceback +from collections.abc import AsyncIterator, Iterator +from typing import Final, Protocol + +from openai import AsyncStream, Stream +from openai.types.audio import ( + TranscriptionStreamEvent, + TranscriptionTextDeltaEvent, + TranscriptionTextDoneEvent, +) + +from litellm.types.utils import TranscriptionResponse + + +class TranscriptionStreamLogging(Protocol): + def success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def handle_sync_success_callbacks_for_async_calls( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + +class _TranscriptionEventCollector: + def __init__(self, duration: float | None) -> None: + self.duration = duration + self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event + self.done_event: TranscriptionTextDoneEvent | None = None + + def add(self, event: TranscriptionStreamEvent) -> None: + if isinstance(event, TranscriptionTextDeltaEvent): + self.text_deltas.append(event.delta) + elif isinstance(event, TranscriptionTextDoneEvent): + self.done_event = event + + def response(self) -> TranscriptionResponse: + done_event: Final = self.done_event + done_languages: Final = getattr(done_event, "languages", None) if done_event is not None else None + response: Final = TranscriptionResponse( + text=done_event.text if done_event is not None else "".join(self.text_deltas), + usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, + languages=( + [ # mutable-ok: the response model requires a concrete serialized language list + language.model_dump() for language in done_languages + ] + if done_languages is not None + else None + ), + ) + if self.duration is not None: + response.set_audio_transcription_duration(self.duration) + return response + + +class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): + def __init__( + self, + stream: Stream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + ) -> None: + self.__dict__.update(stream.__dict__) + self._logging_obj = logging_obj + self._start_time = start_time + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + def _logging_iterator( + self, source_iterator: Iterator[TranscriptionStreamEvent] + ) -> Iterator[TranscriptionStreamEvent]: + try: + for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(exception, traceback.format_exc(), self._start_time, end_time) + raise + finally: + self._finalize() + + def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + self._logging_obj.success_handler( + self._collector.response(), + self._start_time, + datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract + ) + + def close(self) -> None: + try: + super().close() + finally: + self._finalize() + + +class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): + def __init__( + self, + stream: AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + ) -> None: + self.__dict__.update(stream.__dict__) + self._logging_obj = logging_obj + self._start_time = start_time + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + async def _logging_iterator( + self, source_iterator: AsyncIterator[TranscriptionStreamEvent] + ) -> AsyncIterator[TranscriptionStreamEvent]: + try: + async for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(exception, traceback.format_exc(), self._start_time, end_time) + await self._logging_obj.async_failure_handler( + exception, + traceback.format_exc(), + self._start_time, + end_time, + ) + raise + finally: + await self._finalize() + + async def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + response: Final = self._collector.response() + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.handle_sync_success_callbacks_for_async_calls( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + await self._logging_obj.async_success_handler( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + + async def close(self) -> None: + try: + await super().close() + finally: + await self._finalize() + + +def wrap_transcription_stream( + stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, +) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream: + if isinstance(stream, AsyncStream): + return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time) + return LoggingTranscriptionStream(stream, logging_obj, start_time) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 46bf2ec2960..d5a27988762 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -910,6 +910,11 @@ def calculate_cache_writing_cost( class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int cache_hit_audio_tokens: ReadOnly[int] + + cached_text_tokens: ReadOnly[int] + cached_audio_tokens: ReadOnly[int] + cached_image_tokens: ReadOnly[int] + has_cached_tokens_details: ReadOnly[bool] cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None text_tokens: int @@ -996,6 +1001,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, cache_hit_audio_tokens=cached_audio_tokens, + cached_text_tokens=cached_text_tokens, + cached_audio_tokens=cached_audio_tokens, + cached_image_tokens=cached_image_tokens, + has_cached_tokens_details=cached_tokens_details is not None, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, @@ -1079,15 +1088,11 @@ def _calculate_input_cost( prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing - cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] - audio_cache_read_rate: Final = _get_cost_per_unit( - model_info, - _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost - prompt_cost += float(cache_hit_audio_tokens) * ( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost + prompt_cost += _calculate_cache_read_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=model_info, + cache_read_cost=cache_read_cost, + service_tier=service_tier, ) ### AUDIO COST @@ -1167,6 +1172,38 @@ def _calculate_input_cost( return prompt_cost +def _calculate_cache_read_cost( + prompt_tokens_details: PromptTokensDetailsResult, + model_info: ModelInfo, + cache_read_cost: float, + service_tier: str | None, +) -> float: + cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"] + cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"] + cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"] + classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens + unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0) + total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost + float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost + ) + + if cached_audio_tokens: + cached_audio_cost_key: Final = _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier) + cached_audio_cost: Final = _get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached audio contributes to cache-read cost + float(cached_audio_tokens) * float(cached_audio_cost or 0.0) + ) + + if cached_image_tokens: + cached_image_cost_key: Final = _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier) + cached_image_cost: Final = _get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached images contribute to cache-read cost + float(cached_image_tokens) * float(cached_image_cost or 0.0) + ) + + return total_cost + + def _get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: """ Resolve the per-model regional-processing uplift multiplier for a given @@ -1328,6 +1365,10 @@ def generic_cost_per_token( prompt_tokens_details = PromptTokensDetailsResult( cache_hit_tokens=0, cache_hit_audio_tokens=0, + cached_text_tokens=0, + cached_audio_tokens=0, + cached_image_tokens=0, + has_cached_tokens_details=False, cache_creation_tokens=0, cache_creation_token_details=None, text_tokens=usage.prompt_tokens, @@ -1502,6 +1543,7 @@ class BilledTokenRates: cache_creation_input_token_cost: float cache_creation_input_token_cost_above_1hr: float output_cost_per_reasoning_token: float + cache_read_input_image_token_cost: float | None = None def scaled(self, multiplier: float) -> "BilledTokenRates": if multiplier == 1.0: @@ -1514,6 +1556,11 @@ class BilledTokenRates: cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, + cache_read_input_image_token_cost=( + self.cache_read_input_image_token_cost * multiplier + if self.cache_read_input_image_token_cost is not None + else None + ), ) @@ -1617,6 +1664,11 @@ def _cost_map_billed_rates( cache_creation_input_token_cost=cache_creation_cost_rate, cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, output_cost_per_reasoning_token=reasoning_rate, + cache_read_input_image_token_cost=_get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier), + None, + ), ).scaled(multiplier) @@ -1692,6 +1744,12 @@ def get_token_type_cost_breakdown( cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( usage ) + cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"] + image_cache_read_rate: Final = ( + rates.cache_read_input_image_token_cost + if rates.cache_read_input_image_token_cost is not None + else rates.cache_read_input_token_cost + ) cache_creation_cost: Final = ( float(cache_creation_tokens) * rates.cache_creation_input_token_cost if custom_cost_per_token is not None @@ -1705,8 +1763,9 @@ def get_token_type_cost_breakdown( return TokenTypeCostBreakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, cache_read_cost=( - float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost + float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + + float(cached_image_tokens) * image_cache_read_rate ), cache_creation_cost=cache_creation_cost, rates=rates, diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 87524d86c61..c73b74a3c9a 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -38,6 +38,7 @@ from litellm.types.utils import ( StreamingChoices, TextChoices, TextCompletionResponse, + TranscriptionDetectedLanguage, TranscriptionResponse, TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, @@ -772,9 +773,11 @@ def convert_to_model_response_object( model_response_object.data = response_object["data"] if "usage" in response_object and response_object["usage"] is not None: - model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) - model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) - model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) + embedding_usage: Final = model_response_object.usage or Usage() + embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) + embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) + embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0) + model_response_object.usage = embedding_usage if start_time is not None and end_time is not None: model_response_object._response_ms = ( @@ -817,6 +820,12 @@ def convert_to_model_response_object( if key in response_object: setattr(model_response_object, key, response_object[key]) + if "languages" in response_object and response_object["languages"] is not None: + transcription_response: Final = model_response_object + transcription_response.languages = tuple( + TranscriptionDetectedLanguage.model_validate(language) for language in response_object["languages"] + ) + if "usage" in response_object and response_object["usage"] is not None: tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index d2fbb26bb02..5e9e04ae54f 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,4 +1,5 @@ import asyncio +import base64 import json import traceback from collections.abc import Coroutine, Mapping, Sequence @@ -19,6 +20,8 @@ from litellm.types.llms.openai import ( OpenAIRealtimeResponseDelta, OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamSessionEvents, + OpenAIRealtimeTranslationClosedEvent, + OpenAIRealtimeTranslationDurationUsage, ) from litellm.types.realtime import ALL_DELTA_TYPES @@ -137,6 +140,7 @@ class RealTimeStreaming: force_transcription_model: str | None = None, event_normalizer: RealtimeEventNormalizer | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + translation_session: bool = False, ): self.websocket: _ClientWebSocket = websocket self.backend_ws = backend_ws @@ -148,6 +152,10 @@ class RealTimeStreaming: self.input_messages: list[dict[str, str]] = [] self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] + self._is_translation_session = translation_session + self._translation_output_audio_bytes = 0 + self._translation_output_bytes_per_second = 48000.0 + self._translation_usage_finalized = False # Detect whether the client is explicitly opting into the beta protocol. self._client_wants_beta = self._detect_beta_header(websocket) @@ -196,6 +204,7 @@ class RealTimeStreaming: # their input_audio_transcription.completed usage drives duration-based cost. self._force_transcription_model = force_transcription_model self._is_transcription_session: bool = force_transcription_model is not None + self._bound_nested_transcription_model: str | None = None # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer). self._event_normalizer = event_normalizer @@ -410,6 +419,7 @@ class RealTimeStreaming: async def log_messages(self): """Log messages in list""" + self._finalize_translation_usage() if self.logging_obj: if self.input_messages: self.logging_obj.model_call_details["messages"] = self.input_messages @@ -424,6 +434,60 @@ class RealTimeStreaming: ) self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True + def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None: + if not self._is_translation_session: + return + self._capture_translation_output_format(event_obj) + if event_obj.get("type") != "session.output_audio.delta": + return + delta: Final = event_obj.get("delta") + if not isinstance(delta, str): + return + try: + decoded: Final = base64.b64decode(delta, validate=True) + except (ValueError, TypeError): + return + self._translation_output_audio_bytes += len(decoded) + + def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: + session: Final = event_obj.get("session") + if not isinstance(session, dict): + return + audio: Final = session.get("audio") + output: Final = audio.get("output") if isinstance(audio, dict) else None + audio_format: Final = output.get("format") if isinstance(output, dict) else None + if isinstance(audio_format, str): + if audio_format in ("g711_ulaw", "g711_alaw"): + self._translation_output_bytes_per_second = 8000.0 + return + if not isinstance(audio_format, dict): + return + format_type: Final = audio_format.get("type") + rate: Final = audio_format.get("rate") + if not isinstance(rate, (int, float)) or rate <= 0: + return + if format_type == "audio/pcm": + self._translation_output_bytes_per_second = float(rate) * 2 + elif format_type in ("audio/pcmu", "audio/pcma"): + self._translation_output_bytes_per_second = float(rate) + + def _finalize_translation_usage(self) -> None: + if self._translation_usage_finalized: + return + for event in self.messages: + if event.get("type") != "session.closed": + continue + event_usage = event.get("usage") # rebind-ok: each close event carries independent usage + if isinstance(event_usage, dict) and isinstance(event_usage.get("output_seconds"), (int, float)): + self._translation_usage_finalized = True + return + if self._translation_output_audio_bytes == 0: + return + output_seconds: Final = self._translation_output_audio_bytes / self._translation_output_bytes_per_second + synthetic_usage: Final = OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) + self._translation_usage_finalized = True + async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. @@ -436,7 +500,7 @@ class RealTimeStreaming: backend, False if the provider transformation produced no output and the message was effectively dropped. """ - message = self._enforce_transcription_session_model(message) + message = await self._apply_nested_transcription_model_policy(message) if self.provider_config: transformed: Final = self.provider_config.transform_realtime_request( message, self.model, self.session_configuration_request @@ -478,6 +542,90 @@ class RealTimeStreaming: await self.backend_ws.send(message) return True + async def _apply_nested_transcription_model_policy(self, message: str) -> str: + if self._force_transcription_model is not None: + return self._enforce_transcription_session_model(message) + if self._is_translation_session: + return await self._enforce_translation_nested_transcription_model(message) + return message + + def _session_update_message_obj(self, message: str) -> Mapping[str, object] | None: + try: + message_obj: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return None + if message_obj.get("type") not in ( + "session.update", + "transcription_session.update", + ): + return None + return message_obj + + def _nested_transcription_models_from_session( + self, + session: Mapping[str, object], + ) -> tuple[str, ...]: + audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None + nested_model: Final = self._transcription_model_value(nested_transcription) + flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription")) + return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None)) + + def _transcription_model_value(self, transcription_config: object) -> str | None: + if not isinstance(transcription_config, dict): + return None + model: Final = transcription_config.get("model") + if isinstance(model, str) and model: + return model + return None + + def _rewrite_session_update_transcription_model(self, message: str, authorized_model: str) -> str: + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: + return message + session: Final = message_obj.get("session") + if not isinstance(session, dict): + return message + + transcription: Final = session.get("input_audio_transcription") + rewrite_flat: Final = isinstance(transcription, dict) and transcription.get("model") != authorized_model + if isinstance(transcription, dict) and rewrite_flat: + session["input_audio_transcription"] = { + **transcription, + "model": authorized_model, + } + + audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None + rewrite_nested: Final = ( + isinstance(audio, dict) + and isinstance(audio_input, dict) + and isinstance(nested_transcription, dict) + and nested_transcription.get("model") != authorized_model + ) + if ( + isinstance(audio, dict) + and isinstance(audio_input, dict) + and isinstance(nested_transcription, dict) + and rewrite_nested + ): + session["audio"] = { + **audio, + "input": { + **audio_input, + "transcription": { + **nested_transcription, + "model": authorized_model, + }, + }, + } + + if not rewrite_flat and not rewrite_nested: + return message + return json.dumps(message_obj) + def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -495,56 +643,49 @@ class RealTimeStreaming: if self._force_transcription_model is None: return message - try: - message_obj: Final = _decode_json_object(message) - except (json.JSONDecodeError, TypeError): + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message + session: Final = message_obj.get("session") + if isinstance(session, dict) and session.get("type") == "transcription": + self._is_transcription_session = True + return self._rewrite_session_update_transcription_model(message, self._force_transcription_model) - if message_obj.get("type") not in ( - "session.update", - "transcription_session.update", - ): + async def _enforce_translation_nested_transcription_model(self, message: str) -> str: + if self._bound_nested_transcription_model is not None: + return self._rewrite_session_update_transcription_model(message, self._bound_nested_transcription_model) + + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message - session: Final = message_obj.get("session") if not isinstance(session, dict): return message - - if session.get("type") == "transcription": - self._is_transcription_session = True - - authorized_model: Final = self._force_transcription_model - changed = False - - transcription: Final = session.get("input_audio_transcription") - if isinstance(transcription, dict) and transcription.get("model") != authorized_model: - session["input_audio_transcription"] = { - **transcription, - "model": authorized_model, - } - changed = True - - audio: Final = session.get("audio") - if isinstance(audio, dict): - audio_input: Final = audio.get("input") - if isinstance(audio_input, dict): - nested_transcription: Final = audio_input.get("transcription") - if isinstance(nested_transcription, dict) and nested_transcription.get("model") != authorized_model: - session["audio"] = { - **audio, - "input": { - **audio_input, - "transcription": { - **nested_transcription, - "model": authorized_model, - }, - }, - } - changed = True - - if not changed: + nested_models: Final = self._nested_transcription_models_from_session(session) + if not nested_models: return message - return json.dumps(message_obj) + + valid_token: Final = self.user_api_key_dict + if valid_token is None: + return message + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import can_key_call_resolved_model + from litellm.proxy.proxy_server import llm_model_list, llm_router + + if not isinstance(valid_token, UserAPIKeyAuth): + return message + + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + bound_model: Final = nested_models[0] + self._bound_nested_transcription_model = bound_model + return self._rewrite_session_update_transcription_model(message, bound_model) def _uses_deferred_backend_setup(self) -> bool: """True when setup is deferred until the client's first session.update.""" @@ -942,7 +1083,10 @@ class RealTimeStreaming: async def _handle_provider_config_message(self, raw_response: str) -> None: """Process a backend message when a provider_config is set (transformed path).""" - returned_object: Final = self.provider_config.transform_realtime_response( + provider_config: Final = self.provider_config + if provider_config is None: + raise RuntimeError("Provider response handling requires a provider configuration") + returned_object: Final = provider_config.transform_realtime_response( raw_response, self.model, self.logging_obj, @@ -1103,6 +1247,7 @@ class RealTimeStreaming: if self._should_drop_event_from_client(event): continue + self._capture_translation_output_audio(event) if await self._handle_raw_backend_message(event, raw_response): continue @@ -1507,6 +1652,8 @@ class RealTimeStreaming: session = client_event.get("session", {}) if isinstance(session, dict): session = self._remap_beta_session_to_ga(session) + if self._is_translation_session: + session.pop("type", None) msg_obj["session"] = session message = json.dumps(msg_obj) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 564ec94ba6b..fb575a3713c 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,7 +1,7 @@ from collections.abc import Coroutine from typing import TYPE_CHECKING, Any, Final -from openai import AsyncAzureOpenAI, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from pydantic import BaseModel from litellm._uuid import uuid @@ -49,6 +49,7 @@ class AzureAudioTranscription(AzureChatCompletion): timeout=timeout, api_key=api_key, api_base=api_base, + api_version=api_version, client=client, max_retries=max_retries, logging_obj=logging_obj, @@ -66,7 +67,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(azure_client, AzureOpenAI): + if not isinstance(azure_client, (AzureOpenAI, OpenAI)): raise AzureOpenAIError( status_code=500, message="azure_client is not an instance of AzureOpenAI", @@ -85,10 +86,13 @@ class AzureAudioTranscription(AzureChatCompletion): ) response: Final = azure_client.audio.transcriptions.create( - **data, + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) + if data.get("stream") is True: + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -137,7 +141,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(async_azure_client, AsyncAzureOpenAI): + if not isinstance(async_azure_client, (AsyncAzureOpenAI, AsyncOpenAI)): raise AzureOpenAIError( status_code=500, message="async_azure_client is not an instance of AsyncAzureOpenAI", @@ -155,8 +159,15 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) + if data.get("stream") is True: + return await async_azure_client.audio.transcriptions.create( + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, + ) + raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( - **data, timeout=timeout + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, ) headers: Final = dict(raw_response.headers) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index df74975ad0f..393499c0fcd 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -83,6 +83,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version: str | None, realtime_protocol: str | None = None, query_params: RealtimeQueryParams | None = None, + realtime_mode: str = "realtime", ) -> str: """ Construct Azure realtime WebSocket URL. @@ -114,18 +115,26 @@ class AzureOpenAIRealtime(AzureChatCompletion): ) intent: Final = (query_params or {}).get("intent") - if _is_ga: - path = "/openai/v1/realtime" - query_parts = [] - if intent != "transcription" and (query_params is None or "model" in query_params): - query_parts.append(urlencode({"model": model})) - else: - # Default to beta path for backwards compatibility - path = "/openai/realtime" - query_parts = [urlencode({"api-version": api_version, "deployment": model})] + path: Final = ( + "/openai/v1/realtime/translations" + if realtime_mode == "translation" + else "/openai/v1/realtime" + if _is_ga + else "/openai/realtime" + ) + base_query_parts: Final = ( + (urlencode((("model", model),)),) + if realtime_mode == "translation" + else ( + (urlencode((("model", model),)),) + if intent != "transcription" and (query_params is None or "model" in query_params) + else () + ) + if _is_ga + else (urlencode((("api-version", api_version), ("deployment", model))),) + ) - if intent: - query_parts.append(urlencode({"intent": intent})) + query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts qs: Final = "&".join(query_parts) return f"{api_base}{path}?{qs}" if qs else f"{api_base}{path}" @@ -145,6 +154,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", ): import websockets from websockets.asyncio.client import ClientConnection @@ -161,6 +171,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, ) auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token) @@ -184,6 +195,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): force_transcription_model=( model if (query_params or {}).get("intent") == "transcription" else None ), + translation_session=realtime_mode == "translation", ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 633c2b3b6d0..aa2b6d9cda0 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -3,11 +3,16 @@ from typing import Final import litellm +from litellm.constants import AZURE_GA_REALTIME_MODELS from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): + @staticmethod + def _uses_ga_api(model: str, api_version: str | None) -> bool: + return api_version in ("preview", "latest", "v1") or model in AZURE_GA_REALTIME_MODELS + def get_api_base(self, api_base: str | None, **kwargs) -> str: return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or "" @@ -16,6 +21,8 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") + if self._uses_ga_api(model, api_version): + return f"{base}/openai/v1/realtime/client_secrets" version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{base}/openai/realtime/client_secrets?api-version={version}" @@ -25,22 +32,38 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): model: str, api_key: str | None = None, ) -> dict: - return { + validated_headers: Final = { # mutable-ok: provider authentication headers are extended before dispatch **headers, - "api-key": api_key or "", "Content-Type": "application/json", } + if api_key: + validated_headers["api-key"] = api_key + return validated_headers def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") + if self._uses_ga_api(model, api_version): + return f"{base}/openai/v1/realtime/calls" version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{base}/openai/realtime/calls?api-version={version}" def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") + if self._uses_ga_api(model, api_version): + return f"{base}/openai/v1/realtime/transcription_sessions" version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{base}/openai/realtime/transcription_sessions?api-version={version}" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base: Final = self.get_api_base(api_base).rstrip("/") + return f"{base}/openai/v1/realtime/translations/client_secrets" + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base: Final = self.get_api_base(api_base).rstrip("/") + return f"{base}/openai/v1/realtime/translations/calls" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: return { "api-key": ephemeral_key, diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index 43a80edb493..0d64ed3931d 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -63,6 +63,12 @@ class BaseRealtimeHTTPConfig(ABC): base = base.removesuffix("/v1") return f"{base}/v1/realtime/transcription_sessions" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/client_secrets" + @abstractmethod def validate_environment( self, @@ -86,6 +92,10 @@ class BaseRealtimeHTTPConfig(ABC): base: Final = (api_base or "").rstrip("/") return f"{base}/v1/realtime/calls" + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/calls" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: """ Build headers for the realtime_calls POST. diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8cbc28362a8..052b9ee9e87 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -23,7 +23,9 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse import httpx from httpx import USE_CLIENT_DEFAULT from httpx._types import FileContent +from openai import AsyncOpenAI from openai.types.file_deleted import FileDeleted +from openai.types.realtime import RealtimeSessionCreateRequestParam import litellm import litellm.litellm_core_utils @@ -6186,6 +6188,7 @@ class BaseLLMHTTPHandler: "BasePassthroughConfig", "BaseContainerConfig", BaseEvalsAPIConfig, + BaseRealtimeHTTPConfig, ], ): received_status_code: Final = ( @@ -6423,9 +6426,10 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, + use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/client_secrets to upstream provider. @@ -6433,6 +6437,52 @@ class BaseLLMHTTPHandler: Uses provider_config (BaseRealtimeHTTPConfig) for URL construction and header auth when available; falls back to the legacy OpenAI-style defaults. """ + if use_openai_sdk: + trimmed_api_base: Final = api_base.rstrip("/") + normalized_api_base: Final = ( + trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" + ) + owns_client: Final = not isinstance(client, AsyncOpenAI) + openai_client: Final = ( + client + if isinstance(client, AsyncOpenAI) + else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) + ) + logging_obj.pre_call( + input=request_data, + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "complete_input_dict": request_data, + "api_base": normalized_api_base, + }, + ) + try: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + key: str(value) # mutable-ok: SDK headers are materialized as a concrete string mapping + for key, value in (extra_headers or {}).items() # mutable-ok: SDK requires concrete headers + }, + ) + raw_response: Final = await configured_client.post( + "/realtime/client_secrets", + cast_to=httpx.Response, + body=request_data, + ) + response_headers: Final = { # mutable-ok: httpx requires a concrete response-header mapping + key: value # mutable-ok: transport headers are materialized after filtering + for key, value in raw_response.headers.items() # mutable-ok: transport headers are materialized + if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") + } + return httpx.Response( + status_code=raw_response.status_code, + headers=response_headers, + content=raw_response.content, + request=httpx.Request("POST", f"{normalized_api_base}/realtime/client_secrets"), + ) + finally: + if owns_client: + await openai_client.close() return await self._async_realtime_session_post( endpoint="client_secrets", api_base=api_base, @@ -6456,8 +6506,8 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, ) -> httpx.Response: """Forward POST /v1/realtime/transcription_sessions to upstream provider.""" @@ -6475,18 +6525,79 @@ class BaseLLMHTTPHandler: api_version=api_version, ) - async def _async_realtime_session_post( + async def async_realtime_translation_client_secret_handler( self, - endpoint: Literal["client_secrets", "transcription_sessions"], api_base: str, api_key: str, request_data: dict[str, object], logging_obj: LiteLLMLoggingObj, timeout: float | httpx.Timeout, - provider_config: Any | None = None, + provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + api_version: str | None = None, + use_openai_sdk: bool = False, + ) -> httpx.Response: + if use_openai_sdk: + normalized_api_base = api_base.rstrip("/") + if not normalized_api_base.endswith("/v1"): + normalized_api_base = f"{normalized_api_base}/v1" + owns_client: Final = not isinstance(client, AsyncOpenAI) + openai_client: Final = ( + client + if isinstance(client, AsyncOpenAI) + else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) + ) + logging_obj.pre_call( + input=request_data, + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "complete_input_dict": request_data, + "api_base": normalized_api_base, + }, + ) + try: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + key: str(value) for key, value in (extra_headers or {}).items() + }, + ) + return await configured_client.post( + "/realtime/translations/client_secrets", + cast_to=httpx.Response, + body=request_data, + ) + finally: + if owns_client: + await openai_client.close() + return await self._async_realtime_session_post( + endpoint="translation_client_secrets", + api_base=api_base, + api_key=api_key, + request_data=request_data, + logging_obj=logging_obj, + timeout=timeout, + provider_config=provider_config, + model=model, + extra_headers=extra_headers, + client=client, + api_version=api_version, + ) + + async def _async_realtime_session_post( + self, + endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"], + api_base: str, + api_key: str, + request_data: dict[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + provider_config: BaseRealtimeHTTPConfig | None = None, + model: str | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, ) -> httpx.Response: """ @@ -6508,13 +6619,20 @@ class BaseLLMHTTPHandler: url = provider_config.get_transcription_session_url( api_base=api_base, model=model or "", api_version=api_version ) + elif endpoint == "translation_client_secrets": + url = provider_config.get_translation_client_secret_url( + api_base=api_base, model=model or "", api_version=api_version + ) else: url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version) headers: dict[str, object] = provider_config.validate_environment( headers={}, model=model or "", api_key=api_key ) else: - url = f"{api_base.rstrip('/')}/v1/realtime/{endpoint}" + endpoint_path: Final = ( + "translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint + ) + url = f"{api_base.rstrip('/')}/v1/realtime/{endpoint_path}" headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", @@ -6548,6 +6666,80 @@ class BaseLLMHTTPHandler: ) raise + async def _async_realtime_calls_sdk( + self, + api_base: str, + openai_ephemeral_key: str, + sdp_text: str, + session_data: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + extra_headers: Mapping[str, object] | None, + client: object | None, + translation: bool, + ) -> httpx.Response: + normalized_api_base = api_base.rstrip("/") + if not normalized_api_base.endswith("/v1"): + normalized_api_base = f"{normalized_api_base}/v1" + owns_client: Final = not isinstance(client, AsyncOpenAI) + openai_client: Final = ( + client + if isinstance(client, AsyncOpenAI) + else AsyncOpenAI(api_key=openai_ephemeral_key, base_url=normalized_api_base, max_retries=0) + ) + logging_obj.pre_call( + input="realtime_sdp_offer", + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "api_base": normalized_api_base, + "session": session_data, + }, + ) + try: + if translation: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + "Content-Type": "application/sdp", + **{ # mutable-ok: caller headers are normalized into the SDK header mapping + key: str(value) for key, value in (extra_headers or {}).items() + }, + }, + ) + return await configured_client.post( + "/realtime/translations/calls", + cast_to=httpx.Response, + content=sdp_text.encode("utf-8"), + ) + realtime_session_data: Final = cast( # cast-ok: endpoint validation produced an OpenAI realtime session + RealtimeSessionCreateRequestParam, + session_data, + ) + sdk_extra_headers: Final = { # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + key: str(value) for key, value in (extra_headers or {}).items() + } + raw_response: Final = await openai_client.realtime.calls.with_raw_response.create( + sdp=sdp_text, + session=realtime_session_data, + extra_headers=sdk_extra_headers, + timeout=timeout, + ) + return httpx.Response( + status_code=raw_response.status_code, + headers=raw_response.headers, + content=raw_response.content, + request=httpx.Request("POST", f"{normalized_api_base}/realtime/calls"), + ) + finally: + if owns_client: + await openai_client.close() + + @staticmethod + def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: + if isinstance(client, AsyncHTTPHandler): + return client + return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI) + async def async_realtime_calls_handler( self, api_base: str, @@ -6555,12 +6747,14 @@ class BaseLLMHTTPHandler: sdp_body: bytes, logging_obj: LiteLLMLoggingObj, timeout: float | httpx.Timeout, - provider_config: Any | None = None, + provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - session_config: dict[str, object] | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + session_config: Mapping[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, + translation: bool = False, + use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. @@ -6572,18 +6766,45 @@ class BaseLLMHTTPHandler: - sdp: the SDP offer (text) - session: JSON string with {"type": "realtime", "model": "...", ...} """ - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders.OPENAI, + session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally + **( + session_config or {} # mutable-ok: absent session configuration starts from an empty provider payload ) - else: - async_httpx_client = client + } + if "type" not in session_data: + session_data["type"] = "translation" if translation else "realtime" + if "model" not in session_data and model: + session_data["model"] = model + + sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + + if use_openai_sdk: + return await self._async_realtime_calls_sdk( + api_base=api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_text=sdp_text, + session_data=session_data, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=client, + translation=translation, + ) + + async_httpx_client: Final = self._get_realtime_async_http_client(client) if provider_config is not None: - url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) + url = ( + provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version) + if translation + else provider_config.get_realtime_calls_url( + api_base=api_base, model=model or "", api_version=api_version + ) + ) headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key) else: - url = f"{api_base.rstrip('/')}/v1/realtime/calls" + path: Final = "translations/calls" if translation else "calls" + url = f"{api_base.rstrip('/')}/v1/realtime/{path}" headers = { "Authorization": f"Bearer {openai_ephemeral_key}", } @@ -6591,14 +6812,8 @@ class BaseLLMHTTPHandler: if extra_headers: headers.update(extra_headers) - # Build multipart form data: sdp + session JSON - session_data: Final = session_config or {} - if "type" not in session_data: - session_data["type"] = "realtime" - if "model" not in session_data and model: - session_data["model"] = model - - sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + if translation: + headers["Content-Type"] = "application/sdp" files: Final = { "sdp": (None, sdp_text, "text/plain"), @@ -6616,12 +6831,14 @@ class BaseLLMHTTPHandler: ) try: - return await async_httpx_client.post( - url=url, - headers=headers, - files=files, - timeout=timeout, - ) + if translation: + return await async_httpx_client.post( + url=url, + headers=headers, + content=sdp_text, + timeout=timeout, + ) + return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout) except Exception as e: if provider_config is not None: raise self._handle_error( diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index bdc3a6c7908..df2d9ca4c75 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -5,8 +5,17 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ import ssl +from collections.abc import Mapping +from contextlib import AbstractAsyncContextManager +from types import TracebackType from typing import Any, Final, cast +from openai import AsyncOpenAI, omit +from openai.resources.realtime.realtime import ( + AsyncRealtimeConnection, + AsyncRealtimeConnectionManager, +) + from litellm._logging import _redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams @@ -22,6 +31,49 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..openai import OpenAIChatCompletion +class OpenAIRealtimeConnectionAdapter: + def __init__(self, connection: AsyncRealtimeConnection) -> None: + self._connection = connection + + async def send(self, message: str) -> None: + await self._connection.send_raw(message) + + async def recv(self, decode: bool = True) -> str | bytes: + message: Final = await self._connection.recv_bytes() + if decode: + return message.decode("utf-8") + return message + + async def close(self) -> None: + await self._connection.close() + + +class OpenAIRealtimeSDKConnectionManager: + def __init__( + self, + manager: AsyncRealtimeConnectionManager, + owned_client: AsyncOpenAI | None = None, + ) -> None: + self._manager = manager + self._owned_client = owned_client + + async def __aenter__(self) -> OpenAIRealtimeConnectionAdapter: + connection: Final = await self._manager.__aenter__() + return OpenAIRealtimeConnectionAdapter(connection) + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + await self._manager.__aexit__(exc_type, exc, traceback) + finally: + if self._owned_client is not None: + await self._owned_client.close() + + class OpenAIRealtime(OpenAIChatCompletion): """ Base handler for OpenAI-compatible realtime WebSocket connections. @@ -82,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion): return ssl_config - def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str: + def _construct_url(self, api_base: str, query_params: RealtimeQueryParams, realtime_mode: str = "realtime") -> str: """ Construct the backend websocket URL with all query parameters (including 'model'). """ @@ -92,7 +144,8 @@ class OpenAIRealtime(OpenAIChatCompletion): api_base = api_base.replace("http://", "ws://") url = URL(api_base) # Set the correct path - url = url.copy_with(path="/v1/realtime") + path: Final = "/v1/realtime/translations" if realtime_mode == "translation" else "/v1/realtime" + url = url.copy_with(path=path) # Include all query parameters including 'model' if query_params: url = url.copy_with(params=query_params) @@ -106,6 +159,47 @@ class OpenAIRealtime(OpenAIChatCompletion): """ return None + def _create_connection_manager( + self, + api_base: str, + api_key: str, + model: str, + query_params: RealtimeQueryParams, + headers: Mapping[str, str], + timeout: float | None, + realtime_mode: str, + ssl_config: object, + client: object | None, + url: str, + ) -> AbstractAsyncContextManager[object]: + import websockets + + if realtime_mode == "translation" or client is None: + return websockets.connect( + url, + additional_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=ssl_config, + ) + if not isinstance(client, AsyncOpenAI): + raise TypeError("client must be an AsyncOpenAI instance") + openai_client: Final = client + model_query: Final = query_params.get("model") + extra_query: Final = { # mutable-ok: OpenAI SDK accepts a mutable query-parameter mapping + key: value for key, value in query_params.items() if key != "model" + } + sdk_model: Final = omit if query_params.get("intent") == "transcription" else model_query or model + sdk_connection_manager: Final = openai_client.realtime.connect( + model=sdk_model, + extra_query=extra_query, + extra_headers=headers, + websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping + "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + }, + max_retries=0, + ) + return OpenAIRealtimeSDKConnectionManager(sdk_connection_manager) + async def async_realtime( self, model: str, @@ -118,6 +212,7 @@ class OpenAIRealtime(OpenAIChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", **kwargs: object, ): import websockets @@ -131,7 +226,7 @@ class OpenAIRealtime(OpenAIChatCompletion): # Use all query params if provided, else fallback to just model if query_params is None: query_params = {"model": model} - url: Final = self._construct_url(api_base, query_params) + url: Final = self._construct_url(api_base, query_params, realtime_mode=realtime_mode) try: # Get provider-specific SSL configuration @@ -156,15 +251,25 @@ class OpenAIRealtime(OpenAIChatCompletion): "complete_input_dict": {"query_params": query_params}, }, ) - async with websockets.connect( - url, - additional_headers=headers, - max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - ssl=ssl_config, - ) as backend_ws: + connection_manager: Final = self._create_connection_manager( + api_base=api_base, + api_key=api_key, + model=model, + query_params=query_params, + headers=headers, + timeout=timeout, + realtime_mode=realtime_mode, + ssl_config=ssl_config, + client=client, + url=url, + ) + + async with connection_manager as backend_ws: realtime_streaming: Final = RealTimeStreaming( websocket, - cast(ClientConnection, backend_ws), + cast( # cast-ok: both SDK and websockets adapters implement the streaming connection interface + ClientConnection, backend_ws + ), logging_obj, model=model, user_api_key_dict=user_api_key_dict, @@ -173,6 +278,7 @@ class OpenAIRealtime(OpenAIChatCompletion): model if (query_params or {}).get("intent") == "transcription" else None ), event_normalizer=self._make_event_normalizer(), + translation_session=realtime_mode == "translation", ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py index 61dbf20397f..726a9545e11 100644 --- a/litellm/llms/openai/realtime/http_transformation.py +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -27,6 +27,18 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): base = base.removesuffix("/v1") return f"{base}/v1/realtime/transcription_sessions" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/client_secrets" + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/calls" + def validate_environment( self, headers: dict, diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index e7f8dd04516..81f3abd217a 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -14,7 +14,7 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): """ Get the supported OpenAI params for the `gpt-4o-transcribe` models """ - return [ + return [ # mutable-ok: base transcription interface requires a mutable supported-parameter list "language", "prompt", "response_format", @@ -37,3 +37,31 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): return AudioTranscriptionRequestData( data=data, ) + + +class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig): + def get_supported_openai_params( # mutable-ok: base transcription interface returns a mutable parameter list + self, model: str + ) -> list[OpenAIAudioTranscriptionOptionalParams]: + return [ + "prompt", + "response_format", + "keywords", + "languages", + "stream", + ] + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, # mutable-ok: base transformation interface supplies a mutable request payload + litellm_params: dict, # mutable-ok: base transformation interface supplies mutable provider parameters + ) -> AudioTranscriptionRequestData: + data: Final = { # mutable-ok: OpenAI SDK consumes this multipart request mapping + "model": model, + "file": audio_file, + "response_format": "json", + **optional_params, + } + return AudioTranscriptionRequestData(data=data) diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 701b3d30362..2c703a16d4e 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -25,6 +25,23 @@ from ..openai import OpenAIChatCompletion class OpenAIAudioTranscription(OpenAIChatCompletion): # Audio Transcriptions + @staticmethod + def _sdk_compatible_request_data(data: dict) -> dict: + """Route API fields that predate SDK support through ``extra_body``.""" + extension_keys: Final = ("keywords", "languages") + extension_body: Final = {key: data[key] for key in extension_keys if key in data} + if not extension_body: + return data + + existing_extra_body: Final = data.get("extra_body") + return { # mutable-ok: OpenAI SDK requires a mutable request mapping + **{key: value for key, value in data.items() if key not in extension_keys}, + "extra_body": { + **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), + **extension_body, + }, + } + async def make_openai_audio_transcriptions_request( self, openai_aclient: AsyncOpenAI, @@ -37,11 +54,17 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): - call openai_aclient.audio.transcriptions.create by default """ try: - raw_response = await openai_aclient.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) + sdk_data: Final = self._sdk_compatible_request_data(data) + if data.get("stream") is True: + stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout) + return {}, stream_response # mutable-ok: response headers use the existing mutable mapping contract + raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( + **sdk_data, timeout=timeout + ) # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() + parsed_response: Final = raw_response.parse() - return headers, response + return headers, parsed_response except Exception as e: raise e @@ -57,13 +80,17 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): - call openai_aclient.audio.transcriptions.create by default """ try: + sdk_data: Final = self._sdk_compatible_request_data(data) + if data.get("stream") is True: + response = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) + return None, response if litellm.return_response_headers is True: - raw_response = openai_client.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) + raw_response = openai_client.audio.transcriptions.with_raw_response.create(**sdk_data, timeout=timeout) headers: Final = dict(raw_response.headers) response = raw_response.parse() return headers, response else: - response = openai_client.audio.transcriptions.create(**data, timeout=timeout) + response = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) return None, response except Exception as e: raise e @@ -139,6 +166,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, ) + if data.get("stream") is True: + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -200,6 +230,8 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, ) logging_obj.model_call_details["response_headers"] = headers + if data.get("stream") is True: + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() else: diff --git a/litellm/main.py b/litellm/main.py index 7f4b34d28a0..376645f55f4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,6 +37,9 @@ if TYPE_CHECKING: import dotenv import httpx import openai +import tiktoken +from openai import AsyncStream, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel from typing_extensions import overload @@ -7799,7 +7802,10 @@ async def amoderation( @client -async def atranscription(*args, **kwargs) -> TranscriptionResponse: +async def atranscription( + *args, # noqa: ANN002 # public SDK wrapper preserves positional call compatibility + **kwargs, # noqa: ANN003 # kwargs-ok: public SDK wrapper preserves keyword call compatibility +) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: """ Calls openai + azure whisper endpoints. @@ -7832,6 +7838,12 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: else: # Call the synchronous function using run_in_executor response = await loop.run_in_executor(None, func_with_context) + if kwargs.get("stream") is True and isinstance(response, AsyncStream): + if file is not None: + calculated_duration = calculate_request_duration(file) + if calculated_duration is not None: + setattr(response, "_litellm_audio_duration", calculated_duration) + return response if not isinstance(response, TranscriptionResponse): raise ValueError( f"Invalid response from transcription provider, expected TranscriptionResponse, but got {type(response)}" @@ -7844,9 +7856,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: if response is not None and not isinstance(response, Coroutine) and file is not None: existing_duration: Final = getattr(response, "duration", None) if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) - if calculated_duration is not None: - response._hidden_params["audio_transcription_duration"] = calculated_duration + sync_calculated_duration: Final = calculate_request_duration(file) + if sync_calculated_duration is not None: + response.set_audio_transcription_duration(sync_calculated_duration) return response except Exception as e: @@ -7860,16 +7872,52 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: ) +def _validate_gpt_transcription_request( + model: str, + custom_llm_provider: str, + language: str | None, + languages: Sequence[str] | None, + response_format: str | None, + api_version: str | None, +) -> str | None: + if language is not None and languages is not None: + raise litellm.UnsupportedParamsError( + message="language and languages cannot be used together", + model=model, + llm_provider=custom_llm_provider, + ) + if model == "gpt-live-transcribe": + raise litellm.UnsupportedParamsError( + message="gpt-live-transcribe is available through the Realtime API, not file transcription", + model=model, + llm_provider=custom_llm_provider, + ) + if model == "gpt-transcribe" and response_format not in (None, "json"): + raise litellm.UnsupportedParamsError( + message="gpt-transcribe only supports response_format='json'", + model=model, + llm_provider=custom_llm_provider, + ) + if custom_llm_provider == "azure" and model == "gpt-transcribe": + if api_version in (None, "v1", "latest", "preview"): + return litellm.AZURE_DEFAULT_API_VERSION + return api_version + return api_version + + @client def transcription( model: str, file: FileTypes, ## OPTIONAL OPENAI PARAMS ## language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: Literal["json", "text", "srt", "verbose_json", "vtt"] | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, temperature: int | None = None, # openai defaults this to 0 + stream: bool | None = None, ## LITELLM PARAMS ## user: str | None = None, timeout=600, # default to 10 minutes @@ -7879,7 +7927,11 @@ def transcription( max_retries: int | None = None, custom_llm_provider=None, **kwargs, -) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: +) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] +): """ Calls openai + azure whisper endpoints. @@ -7917,13 +7969,25 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key + validated_api_version: Final = _validate_gpt_transcription_request( + model=model, + custom_llm_provider=custom_llm_provider, + language=language, + languages=languages, + response_format=response_format, + api_version=api_version, + ) + optional_params: Final = get_optional_params_transcription( model=model, language=language, + languages=languages, + keywords=keywords, prompt=prompt, response_format=response_format, timestamp_granularities=timestamp_granularities, temperature=temperature, + stream=stream, custom_llm_provider=custom_llm_provider, **non_default_params, ) @@ -7946,7 +8010,13 @@ def transcription( custom_llm_provider=custom_llm_provider, ) - response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None + response: ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | AsyncStream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + | None + ) = None provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model, @@ -7961,7 +8031,7 @@ def transcription( # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + azure_api_version: Final = validated_api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN") @@ -7980,7 +8050,7 @@ def transcription( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - api_version=api_version, + api_version=azure_api_version, azure_ad_token=azure_ad_token, max_retries=max_retries, litellm_params=litellm_params_dict, @@ -8114,11 +8184,12 @@ def transcription( # Store duration in _hidden_params for cost calculation without # exposing it in the response body (see sync path comment above). if response is not None and not isinstance(response, Coroutine): - existing_duration: Final = getattr(response, "duration", None) - if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) + calculated_duration: Final = calculate_request_duration(file) + if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - response._hidden_params["audio_transcription_duration"] = calculated_duration + setattr(response, "_litellm_audio_duration", calculated_duration) + elif getattr(response, "duration", None) is None and calculated_duration is not None: + response.set_audio_transcription_duration(calculated_duration) if response is None: raise ValueError("Unmapped provider passed in. Unable to get the response.") diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 5be87a8bf4d..ed09149904e 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -31,6 +31,20 @@ def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], return _register +def _include_router_prepend(attr_name: str = "router") -> Callable[["FastAPI", object], None]: + def _register(app: "FastAPI", module: object) -> None: + router: Final = app.router + route_count: Final = len(router.routes) + app.include_router(getattr(module, attr_name)) + new_routes: Final = router.routes[route_count:] + router.routes[:] = [ # mutable-ok: FastAPI route registration requires in-place list replacement + *new_routes, + *router.routes[:route_count], + ] + + return _register + + def _mount_app(prefix: str, attr_name: str = "app") -> Callable[["FastAPI", object], None]: def _register(app: "FastAPI", module: object) -> None: app.mount(path=prefix, app=getattr(module, attr_name)) @@ -225,6 +239,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( name="realtime", module_path="litellm.proxy.realtime_endpoints.endpoints", path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"), + register_fn=_include_router_prepend(), ), LazyFeature( name="anthropic_passthrough", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 71dbf0da239..9a64be44330 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -20245,6 +20245,51 @@ ] } }, + "/openai/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "llm_passthrough" + ] + } + }, + "/openai/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "llm_passthrough" + ] + } + }, "/openai/v1/responses": { "post": { "description": "Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses\n\nSupports background mode with polling_via_cache for partial response retrieval.\nWhen background=true and polling_via_cache is enabled, returns a polling_id immediately\nand streams the response in the background, updating Redis cache.\n\n```bash\n# Normal request\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer sk-1234\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\"\n}'\n\n# Background request with polling\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer sk-1234\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\",\n \"background\": true\n}'\n```", @@ -39286,6 +39331,51 @@ ] } }, + "/openai/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post_2", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/openai/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post_2", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/realtime/calls": { "post": { "operationId": "proxy_realtime_calls_realtime_calls_post", @@ -39358,6 +39448,51 @@ ] } }, + "/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/v1/realtime/calls": { "post": { "operationId": "proxy_realtime_calls_v1_realtime_calls_post", @@ -39429,6 +39564,51 @@ "realtime" ] } + }, + "/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_v1_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_v1_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 54574ed64e3..de0eb77d762 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -417,6 +417,9 @@ class LiteLLMRoutes(enum.Enum): "/realtime?{model}", "/v1/realtime?{model}", "/openai/v1/realtime?{model}", + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", # realtime (GA WebRTC HTTP routes) "/realtime/client_secrets", "/v1/realtime/client_secrets", @@ -427,6 +430,12 @@ class LiteLLMRoutes(enum.Enum): "/realtime/transcription_sessions", "/v1/realtime/transcription_sessions", "/openai/v1/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", # responses API "/responses", "/v1/responses", @@ -2625,6 +2634,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): admission_queue_timeout_seconds: float = Field( 1.0, gt=0, description="maximum time a request waits for a worker slot" ) + allow_non_billable_realtime_protocols: bool = Field( + False, + description="Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement", + ) plugins: list[PluginConfig] | None = Field( None, description="external services registered as embeddable UI plugins" ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c40090233be..991da2b200f 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -132,7 +132,9 @@ ProxyRouteType: TypeAlias = Literal[ "_arealtime", "_aresponses_websocket", "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "aget_responses", "adelete_responses", "acancel_responses", diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1c2bd7ea217..37805ff0459 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -399,14 +399,18 @@ async def get_form_data(request: Request) -> dict[str, Any]: Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` """ form: Final = await request.form() - parsed_form_data: Final[dict[str, Any]] = {} - for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat - if key.endswith("[]"): - clean_key = key[:-2] - parsed_form_data.setdefault(clean_key, []).append(value) - else: - parsed_form_data[key] = value - return parsed_form_data + form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items()) + array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]")) + normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items) + normalized_keys: Final = frozenset(key for key, _ in normalized_items) + return { # mutable-ok: request parsers expose a mutable form-data mapping to endpoint handlers + key: [ # mutable-ok: repeated multipart values follow the established mutable list contract + value for item_key, value in normalized_items if item_key == key + ] + if key in array_keys + else next(value for item_key, value in reversed(normalized_items) if item_key == key) + for key in normalized_keys + } async def convert_upload_files_to_file_data( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 92a75bf953a..6b95865c8ff 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -49,6 +49,7 @@ from typing import ( import anyio import websockets import websockets.exceptions +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError from pydantic.fields import FieldInfo, PydanticUndefined from typing_extensions import NotRequired, ReadOnly, assert_never @@ -12199,7 +12200,11 @@ async def audio_transcriptions( try: # Use orjson to parse JSON data, orjson speeds up requests significantly form_data: Final = await get_form_data(request) - data = {key: value for key, value in form_data.items() if key != "file"} | data + data = { + key: value is True or str(value).lower() in ("1", "true") if key == "stream" else value + for key, value in form_data.items() + if key != "file" + } | data # Include original request and headers in the data data = await add_litellm_data_to_request( @@ -12265,6 +12270,29 @@ async def audio_transcriptions( finally: file_object.close() # close the file read in by io library + if data.get("stream") is True: + if not hasattr(response, "__aiter__"): + raise TypeError(f"Streaming transcription returned {type(response).__name__}, expected an async stream") + stream_response: Final = cast(AsyncIterator[TranscriptionStreamEvent], response) + + async def transcription_event_stream( + stream: AsyncIterator[TranscriptionStreamEvent], + ) -> AsyncGenerator[str, None]: + try: + async for event in stream: + yield f"data: {event.model_dump_json()}\n\n" + finally: + close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None) + if callable(close): + close_result: Final = close() + if inspect.isawaitable(close_result): + await close_result + + return StreamingResponse( + transcription_event_stream(stream_response), + media_type="text/event-stream", + ) + ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) @@ -12431,9 +12459,39 @@ async def _reject_realtime_session( await _release_realtime_max_parallel_slot(user_api_key_dict) +def _resolve_realtime_route_model( + model: str | None, + intent: str | None, + is_translation: bool, +) -> str | None: + if model is not None: + return model + if is_translation: + return "gpt-realtime-translate" + if intent == "transcription": + return "gpt-realtime-whisper" + return None + + +def _resolve_realtime_upstream_query_model( + model: str | None, + intent: str | None, + is_translation: bool, + route_model: str, +) -> str | None: + if intent == "transcription": + return None + if is_translation: + return route_model + return model + + @app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") +@app.websocket("/openai/v1/realtime/translations") +@app.websocket("/v1/realtime/translations") +@app.websocket("/realtime/translations") async def realtime_websocket_endpoint( websocket: WebSocket, model: str | None = fastapi.Query(None, description="The model to use for the websocket connection."), @@ -12451,15 +12509,13 @@ async def realtime_websocket_endpoint( if requested_protocols: accept_kwargs["subprotocol"] = requested_protocols[0] - route_model = model + is_translation: Final = websocket.url.path.endswith("/realtime/translations") + route_model: Final = _resolve_realtime_route_model(model, intent, is_translation) if route_model is None: - if intent == "transcription": - route_model = "gpt-realtime-whisper" - else: - await _reject_realtime_session( - websocket, user_api_key_dict, code=1008, reason="model query parameter is required" - ) - return + await _reject_realtime_session( + websocket, user_api_key_dict, code=1008, reason="model query parameter is required" + ) + return assert route_model is not None try: await can_key_call_resolved_model( @@ -12475,12 +12531,24 @@ async def realtime_websocket_endpoint( await websocket.accept(**accept_kwargs) # Only use explicit parameters, not all query params - query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent))) + query_model: Final = _resolve_realtime_upstream_query_model( + model=model, + intent=intent, + is_translation=is_translation, + route_model=route_model, + ) + query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys + RealtimeQueryParams, + dict( # mutable-ok: downstream realtime routing normalizes this request-scoped query mapping + _realtime_query_params_template(query_model, intent) + ), + ) data: dict[str, object] = { "model": route_model, "websocket": websocket, "query_params": query_params, # Only explicit params + "realtime_mode": "translation" if is_translation else "realtime", } # Pass guardrails into data so pre-call guardrail processing picks them up diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index be2ac2ff33e..bee2a55f5b5 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -2,6 +2,7 @@ import json import time +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -37,7 +38,21 @@ router: Final = APIRouter() _REALTIME_TOKEN_VERSION: Final = "realtime_v1" _DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview" _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" -_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription") +_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation") +_NON_BILLABLE_REALTIME_PROTOCOL_SETTING: Final = "allow_non_billable_realtime_protocols" +_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE: Final = ( + "Realtime WebRTC endpoints are disabled because provider usage bypasses LiteLLM billing. " + "Set general_settings.allow_non_billable_realtime_protocols to true to opt in" +) + + +def _enforce_non_billable_realtime_protocol_gate(general_settings: Mapping[str, object]) -> None: + if general_settings.get(_NON_BILLABLE_REALTIME_PROTOCOL_SETTING) is True: + return + raise HTTPException( + status_code=http_status.HTTP_403_FORBIDDEN, + detail=_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE, + ) def _coerce_realtime_session_type(session_type: str | None) -> str: @@ -120,19 +135,47 @@ def _set_transcription_model_on_session( } +async def _authorize_and_bind_nested_transcription_models( + session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization + user_api_key_dict: UserAPIKeyAuth, + llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list + llm_router: Any, +) -> None: + nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data)) + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + if nested_models: + _set_transcription_model_on_session( + session=session_data, + model=nested_models[0], + ) + + async def _prepare_client_secret_session( req: RealtimeClientSecretRequest, user_api_key_dict: UserAPIKeyAuth, llm_model_list: list | None, llm_router: "Router | None", + forced_session_type: str | None = None, ) -> tuple[str, dict | None, str]: - session_type: Final = _coerce_realtime_session_type(req.session.type if req.session else None) - session_data: Final[dict | None] = req.session.model_dump(exclude_none=True) if req.session else None + requested_session_type: Final = req.session.type if req.session else None + if forced_session_type is None and requested_session_type == "translation": + raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint") + session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type) + session_data: Final[dict | None] = ( + req.session.model_dump(exclude_none=True) if req.session else ({} if session_type == "translation" else None) + ) if session_data is not None: session_data["type"] = session_type session_model: Final = req.session.model if req.session else None - model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL + default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL + model: str = session_model or req.model or default_model if session_type != "transcription": await can_key_call_resolved_model( model=model, @@ -140,6 +183,15 @@ async def _prepare_client_secret_session( llm_model_list=llm_model_list, llm_router=llm_router, ) + if session_data is not None: + session_data["model"] = model + if session_type == "translation": + await _authorize_and_bind_nested_transcription_models( + session_data=session_data, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) return model, session_data, session_type transcription_model_candidates: Final = _transcription_model_candidates_from_session(session_data or {}) @@ -228,6 +280,21 @@ def _decode_realtime_token_payload( dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/openai/v1/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) async def create_realtime_client_secret( request: Request, fastapi_response: Response, @@ -245,16 +312,19 @@ async def create_realtime_client_secret( version, ) + _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await _read_request_body(request=request) req: Final = RealtimeClientSecretRequest(**body) + is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path model, session_data, session_type = await _prepare_client_secret_session( req=req, user_api_key_dict=user_api_key_dict, llm_model_list=llm_model_list, llm_router=llm_router, + forced_session_type="translation" if is_translation_request else None, ) data = {"model": model} @@ -278,17 +348,20 @@ async def create_realtime_client_secret( proxy_config=proxy_config, ) + call_type: Final = ( + "acreate_realtime_translation_client_secret" if is_translation_request else "acreate_realtime_client_secret" + ) data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=data, - call_type="acreate_realtime_client_secret", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/client_secrets (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="acreate_realtime_client_secret", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -371,6 +444,18 @@ async def create_realtime_client_secret( "/openai/v1/realtime/calls", tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/openai/v1/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) async def proxy_realtime_calls( request: Request, fastapi_response: Response, @@ -396,6 +481,7 @@ async def proxy_realtime_calls( media_type="application/json", ) + is_translation_request: Final = "/realtime/translations/calls" in request.url.path encrypted_token: Final = auth_header.removeprefix("Bearer ").strip() decrypted_token_value: Final = decrypt_value_helper( value=encrypted_token, @@ -408,26 +494,42 @@ async def proxy_realtime_calls( media_type="application/json", ) + _enforce_non_billable_realtime_protocol_gate(general_settings) sdp_body: Final[bytes] = await request.body() decoded_payload: Final = _decode_realtime_token_payload(decrypted_token_value) if decoded_payload is not None: # Check token expiry expires_at: Final = decoded_payload.get("expires_at") - if expires_at is not None and isinstance(expires_at, int): - if time.time() > expires_at: - return Response( - content=json.dumps({"error": "Token has expired"}), - status_code=http_status.HTTP_401_UNAUTHORIZED, - media_type="application/json", - ) + if isinstance(expires_at, int) and time.time() > expires_at: + return Response( + content=json.dumps({"error": "Token has expired"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decoded_payload.get("ephemeral_key", "") model = decoded_payload.get("model_id") or request.query_params.get("model") or _DEFAULT_REALTIME_MODEL user_id = decoded_payload.get("user_id") or None team_id = decoded_payload.get("team_id") or None - session_type = _coerce_realtime_session_type(decoded_payload.get("session_type")) + raw_session_type: Final = decoded_payload.get("session_type") + session_type = _coerce_realtime_session_type(raw_session_type) + if is_translation_request != (raw_session_type == "translation"): + return Response( + content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping + {"error": "Token is not valid for this Realtime endpoint"} + ), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) else: - # Backward compatibility: older tokens contained only encrypted upstream key. + if is_translation_request: + return Response( + content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping + {"error": "Token is not valid for this Realtime endpoint"} + ), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decrypted_token_value model = request.query_params.get("model", _DEFAULT_REALTIME_MODEL) user_id = None @@ -471,17 +573,18 @@ async def proxy_realtime_calls( proxy_config=proxy_config, ) + call_type: Final = "arealtime_translation_calls" if is_translation_request else "arealtime_calls" data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=minimal_auth, data=data, - call_type="arealtime_calls", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/calls (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="arealtime_calls", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -557,6 +660,7 @@ async def create_realtime_transcription_session( version, ) + _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await _read_request_body(request=request) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 536c58df65a..399ecc19d27 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -342,7 +342,9 @@ RouteType = Literal[ "alist_input_items", "_arealtime", # private function for realtime API "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "acreate_realtime_transcription_session", "_aresponses_websocket", # private function for responses WebSocket mode "aimage_edit", diff --git a/litellm/realtime_api/README.md b/litellm/realtime_api/README.md index d810de2f24f..d9699f57da7 100644 --- a/litellm/realtime_api/README.md +++ b/litellm/realtime_api/README.md @@ -6,4 +6,16 @@ Supported endpoints: Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI. -For user-facing documentation and usage examples, see the litellm-docs repo. \ No newline at end of file +Billing visibility: +- WebSocket sessions pass provider usage events through LiteLLM and support local spend tracking +- Client-secret and SDP call endpoints only proxy session setup; subsequent WebRTC media and usage events travel over the peer connection, so LiteLLM cannot record inference spend or enforce spend-based budgets for those sessions +- Use the proxied WebSocket transport when LiteLLM spend logs and budgets must include Realtime inference + +Non-billable Realtime protocols are disabled by default. Operators who accept the billing and budget-enforcement limitation can opt in: + +```yaml +general_settings: + allow_non_billable_realtime_protocols: true +``` + +For user-facing documentation and usage examples, see the litellm-docs repo. diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index acc42c44c04..02055988565 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -6,14 +6,18 @@ from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast +import httpx + import litellm from litellm.constants import ( + AZURE_GA_REALTIME_MODELS, AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.xai.common_utils import XAIModelInfo @@ -70,6 +74,30 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: + if session.get("type") == "transcription": + audio = session.get("audio") + audio = audio if isinstance(audio, dict) else {} # mutable-ok: nested session model is rebuilt locally + audio_input = audio.get("input") + audio_input = ( # mutable-ok: nested session model is rebuilt locally + audio_input if isinstance(audio_input, dict) else {} + ) + transcription = audio_input.get("transcription") + transcription = ( # mutable-ok: nested session model is rebuilt locally + transcription if isinstance(transcription, dict) else {} + ) + return { # mutable-ok: provider routing requires an independently mutable session payload + **session, + "audio": { # mutable-ok: provider routing rebuilds nested audio configuration + **audio, + "input": { # mutable-ok: provider routing rebuilds nested input configuration + **audio_input, + "transcription": { # mutable-ok: resolved deployment replaces only the transcription model + **transcription, + "model": model_name, + }, + }, + }, + } if "model" not in session: return session return {**session, "model": model_name} @@ -84,6 +112,21 @@ def _build_litellm_metadata(kwargs: dict) -> dict: return metadata +def _resolve_azure_realtime_protocol( + model: str, + realtime_protocol: str | None, + query_params: RealtimeQueryParams | None, + realtime_mode: str, +) -> str: + if model in AZURE_GA_REALTIME_MODELS: + if realtime_protocol is not None and realtime_protocol.upper() not in ("GA", "V1"): + raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") + return "GA" + if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": + return "GA" + return realtime_protocol or "beta" + + def _get_realtime_http_provider_config( custom_llm_provider: str, dynamic_api_base: str | None, @@ -97,10 +140,6 @@ def _get_realtime_http_provider_config( Uses ProviderConfigManager so each provider keeps its credential-resolution and URL-construction logic in its own transformation class. """ - from litellm.llms.base_llm.realtime.http_transformation import ( - BaseRealtimeHTTPConfig, - ) - provider_config: BaseRealtimeHTTPConfig | None = None if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_realtime_http_config( @@ -124,11 +163,27 @@ def _get_realtime_http_provider_config( return provider_config, resolved_api_base.rstrip("/"), resolved_api_key +def _get_realtime_http_extra_headers( + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + resolved_api_key: str, + extra_headers: Mapping[str, object] | None, +) -> Mapping[str, object] | None: + resolved_headers: Final = { # mutable-ok: Azure authentication may extend caller-supplied headers + **(extra_headers or {}) + } + if custom_llm_provider == "azure" and not resolved_api_key: + azure_ad_token: Final = get_azure_ad_token(litellm_params) + if azure_ad_token: + resolved_headers["Authorization"] = f"Bearer {azure_ad_token}" + return resolved_headers or None + + @wrapper_client async def acreate_realtime_client_secret( model: str | None = None, - session: dict[str, Any] | None = None, - expires_after: dict[str, Any] | None = None, + session: Mapping[str, Any] | None = None, + expires_after: Mapping[str, Any] | None = None, timeout: float | None = None, **kwargs, ): @@ -137,30 +192,48 @@ async def acreate_realtime_client_secret( session=RealtimeSessionConfig(**session) if session else None, expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None, ) - model_name = (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview" + transcription_model: Final = ( + req.session.audio.input.transcription.model + if req.session is not None + and req.session.audio is not None + and req.session.audio.input is not None + and req.session.audio.input.transcription is not None + else None + ) + provider_qualified_model: Final = ( + req.model + if req.model is not None + and "/" in req.model + and req.model.split("/", 1)[0] in LlmProviders._member_map_.values() + else None + ) + requested_model_name: Final = ( + provider_qualified_model + or transcription_model + or (req.session.model if req.session is not None else None) + or req.model + or "gpt-4o-realtime-preview" + ) litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj") litellm_params: Final = GenericLiteLLMParams(**kwargs) - ( - model_name, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = get_llm_provider( - model=model_name, + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) - ( - provider_config, - resolved_api_base, - resolved_api_key, - ) = _get_realtime_http_provider_config( + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( custom_llm_provider=custom_llm_provider, dynamic_api_base=dynamic_api_base, dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -171,6 +244,11 @@ async def acreate_realtime_client_secret( request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) if isinstance(request_data.get("session"), dict): request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + elif req.model is not None: + request_data["session"] = { # mutable-ok: OpenAI SDK consumes this request-scoped session payload + "type": "realtime", + "model": model_name, + } return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=resolved_api_key, @@ -179,9 +257,86 @@ async def acreate_realtime_client_secret( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", + ) + + +@wrapper_client +async def acreate_realtime_translation_client_secret( + model: str | None = None, + session: Mapping[str, Any] | None = None, + expires_after: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options +) -> httpx.Response: + requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate" + session_config: Final = RealtimeSessionConfig.model_validate( + { # mutable-ok: Pydantic validates this request-scoped translation session payload + **(session or {}), + "type": "translation", + "model": requested_model_name, + } + ) + req: Final = RealtimeClientSecretRequest( + model=requested_model_name, + session=session_config, + expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None, + ) + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ # mutable-ok: logging owns a mutable request metadata payload + "expires_after": expires_after, + "session": session, + }, + litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload + "api_base": resolved_api_base + }, + custom_llm_provider=custom_llm_provider, + ) + request_data: Final = req.model_dump( + exclude_none=True, + exclude={"model"}, # mutable-ok: Pydantic requires a mutable field-exclusion set + ) + request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + request_data["session"].pop("type", None) + return await base_llm_http_handler.async_realtime_translation_client_secret_handler( + api_base=resolved_api_base, + api_key=resolved_api_key, + request_data=request_data, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + extra_headers=resolved_extra_headers, + client=kwargs.get("client"), + api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", ) @@ -229,6 +384,12 @@ async def acreate_realtime_transcription_session( dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -251,7 +412,7 @@ async def acreate_realtime_transcription_session( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, ) @@ -307,6 +468,70 @@ async def arealtime_calls( extra_headers=kwargs.get("extra_headers"), client=kwargs.get("client"), api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", + ) + + +@wrapper_client +async def arealtime_translation_calls( + openai_ephemeral_key: str, + sdp_body: bytes, + model: str | None = None, + session: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options +) -> httpx.Response: + requested_model_name: Final = model or "gpt-realtime-translate" + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + session_config: Final = _with_resolved_session_model( + { # mutable-ok: provider routing requires an independently mutable session payload + **(session or {}), + "type": "translation", + "model": model_name, + }, + model_name, + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ # mutable-ok: logging owns a mutable request metadata payload + "realtime_translation_calls": True, + "session": session_config, + }, + litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload + "api_base": resolved_api_base + }, + custom_llm_provider=custom_llm_provider, + ) + return await base_llm_http_handler.async_realtime_calls_handler( + api_base=resolved_api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_body=sdp_body, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + session_config=session_config, + extra_headers=kwargs.get("extra_headers"), + client=kwargs.get("client"), + api_version=litellm_params.api_version, + translation=True, + use_openai_sdk=custom_llm_provider == "openai", ) @@ -356,6 +581,7 @@ async def _arealtime( client: object | None = None, timeout: float | None = None, query_params: RealtimeQueryParams | None = None, + realtime_mode: str = "realtime", **kwargs, ): """ @@ -423,6 +649,9 @@ async def _arealtime( api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # set API KEY api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_API_KEY") + resolved_azure_ad_token = azure_ad_token or litellm_params.azure_ad_token + if not api_key and not resolved_azure_ad_token: + resolved_azure_ad_token = get_azure_ad_token(litellm_params) api_version = api_version or litellm_params.api_version or "2024-10-01-preview" @@ -431,11 +660,17 @@ async def _arealtime( or litellm_params.get("realtime_protocol") or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") ) - realtime_protocol: Final = azure_realtime_protocol_for_client( - configured_realtime_protocol, query_params=query_params, websocket=websocket - ) - resolved_azure_ad_token: Final = ( - None if api_key else get_azure_ad_token(GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token)) + realtime_protocol: Final = _resolve_azure_realtime_protocol( + model=model, + realtime_protocol=( + configured_realtime_protocol + if model in AZURE_GA_REALTIME_MODELS or realtime_mode == "translation" + else azure_realtime_protocol_for_client( + configured_realtime_protocol, query_params=query_params, websocket=websocket + ) + ), + query_params=query_params, + realtime_mode=realtime_mode, ) await azure_realtime.async_realtime( model=model, @@ -449,6 +684,7 @@ async def _arealtime( logging_obj=litellm_logging_obj, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), litellm_metadata=_build_litellm_metadata(kwargs), ) @@ -463,9 +699,10 @@ async def _arealtime( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - client=None, + client=client, timeout=timeout, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), litellm_metadata=_build_litellm_metadata(kwargs), ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 59655800af6..044e15339a5 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -176,6 +176,8 @@ _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( "insufficient_quota": 429, "vector_store_timeout": 504, "invalid_prompt": 400, + "data_residency_mismatch": 400, + "bio_policy": 400, "invalid_image": 400, "invalid_image_format": 400, "invalid_base64_image": 400, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index a2642795cea..d94b9940f24 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1188,12 +1188,14 @@ class ResponseAPILoggingUtils: else: prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), + cached_tokens_details=getattr( + response_api_usage.input_tokens_details, + "cached_tokens_details", + None, + ), audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), - cached_tokens_details=getattr( - response_api_usage.input_tokens_details, "cached_tokens_details", None - ), video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), diff --git a/litellm/router.py b/litellm/router.py index 7267f6eb3ba..ba35ebebe88 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1893,6 +1893,14 @@ class Router: self.acreate_realtime_transcription_session = self.factory_function( litellm.acreate_realtime_transcription_session, call_type="acreate_realtime_transcription_session" ) + self.acreate_realtime_translation_client_secret = self.factory_function( + litellm.acreate_realtime_translation_client_secret, + call_type="acreate_realtime_translation_client_secret", + ) + self.arealtime_translation_calls = self.factory_function( + litellm.arealtime_translation_calls, + call_type="arealtime_translation_calls", + ) self._aresponses_websocket = self.factory_function( litellm._aresponses_websocket, call_type="_aresponses_websocket" ) @@ -6516,6 +6524,8 @@ class Router: "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", "_aresponses_websocket", "acreate_fine_tuning_job", "acancel_fine_tuning_job", @@ -6777,6 +6787,8 @@ class Router: "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 599b1a76249..083b7beda41 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1154,9 +1154,12 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]] OpenAIAudioTranscriptionOptionalParams = Literal[ "language", + "languages", + "keywords", "prompt", "temperature", "response_format", + "stream", "timestamp_granularities", "include", ] @@ -2308,6 +2311,16 @@ class OpenAIRealtimeResponseUsage(TypedDict): output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] +class OpenAIRealtimeTranslationDurationUsage(TypedDict): + type: ReadOnly[Literal["duration"]] + output_seconds: ReadOnly[float] + + +class OpenAIRealtimeTranslationClosedEvent(TypedDict): + type: ReadOnly[Literal["session.closed"]] + usage: ReadOnly[OpenAIRealtimeTranslationDurationUsage] + + class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" # Beta delta event names @@ -2350,6 +2363,7 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeInputAudioTranscriptionCompleted | OpenAIRealtimeTranscriptionSessionCreated | OpenAIRealtimeErrorEvent + | OpenAIRealtimeTranslationClosedEvent ) OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents] diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 855cccc8ddd..f6ab5fc31db 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,6 +1,7 @@ -from typing import Any, Literal +from collections.abc import Mapping, Sequence +from typing import Any, Final, Literal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from typing_extensions import ReadOnly, TypedDict from .llms.openai import ( @@ -10,6 +11,7 @@ from .llms.openai import ( ) ALL_DELTA_TYPES = Literal["text", "audio"] +RealtimeSessionType: Final = Literal["realtime", "transcription", "translation"] class RealtimeResponseTransformInput(TypedDict): @@ -64,6 +66,41 @@ class RealtimeExpiresAfter(BaseModel): seconds: int | None = None +class RealtimeAudioTranscriptionConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + model: str | None = None + delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None + keywords: Sequence[str] | None = None + language: str | None = None + languages: Sequence[str] | None = None + prompt: str | None = None + + +class RealtimeAudioInputConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + format: str | Mapping[str, object] | None = None + noise_reduction: Mapping[str, object] | None = None + transcription: RealtimeAudioTranscriptionConfig | None = None + turn_detection: Mapping[str, object] | None = None + + +class RealtimeAudioOutputConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + format: str | Mapping[str, object] | None = None + language: str | None = None + voice: str | Mapping[str, object] | None = None + + +class RealtimeSessionAudioConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + input: RealtimeAudioInputConfig | None = None + output: RealtimeAudioOutputConfig | None = None + + class RealtimeSessionConfig(BaseModel): """ Session configuration nested inside the client_secrets request body. @@ -75,10 +112,10 @@ class RealtimeSessionConfig(BaseModel): model_config = {"extra": "allow"} - type: str | None = None + type: RealtimeSessionType | None = None model: str | None = None instructions: str | None = None - audio: dict[str, object] | None = None + audio: RealtimeSessionAudioConfig | None = None include: list[str] | None = None max_output_tokens: int | str | None = None output_modalities: list[str] | None = None @@ -132,12 +169,15 @@ class RealtimeTranscriptionSessionRequest(BaseModel): # LiteLLM-only routing hint — stripped before forwarding upstream. model: str | None = None input_audio_transcription: dict[str, Any] | None = None + audio: RealtimeSessionAudioConfig | None = None def resolved_model(self) -> str | None: if self.model: return self.model if self.input_audio_transcription: return self.input_audio_transcription.get("model") + if self.audio and self.audio.input and self.audio.input.transcription: + return self.audio.input.transcription.model return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index dfc98a9d89d..6667e2ee400 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -449,6 +449,11 @@ class CallTypes(str, Enum): asearch = "asearch" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" + acreate_realtime_client_secret = "acreate_realtime_client_secret" + arealtime_calls = "arealtime_calls" + acreate_realtime_transcription_session = "acreate_realtime_transcription_session" + acreate_realtime_translation_client_secret = "acreate_realtime_translation_client_secret" + arealtime_translation_calls = "arealtime_translation_calls" create_batch = "create_batch" acreate_batch = "acreate_batch" aretrieve_batch = "aretrieve_batch" @@ -677,10 +682,18 @@ CallTypesLiteral = Literal[ "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ] # Mapping of API routes to their corresponding call types API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { + "/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/openai/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/openai/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), # Chat Completions "/chat/completions": [CallTypes.acompletion, CallTypes.completion], "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], @@ -993,9 +1006,12 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API - "/realtime": [CallTypes.arealtime], - "/v1/realtime": [CallTypes.arealtime], - "/openai/v1/realtime": [CallTypes.arealtime], + "/realtime": (CallTypes.arealtime,), + "/v1/realtime": (CallTypes.arealtime,), + "/openai/v1/realtime": (CallTypes.arealtime,), + "/realtime/translations": (CallTypes.arealtime,), + "/v1/realtime/translations": (CallTypes.arealtime,), + "/openai/v1/realtime/translations": (CallTypes.arealtime,), # Provider-specific routes "/anthropic/v1/messages": [CallTypes.anthropic_messages], # Google GenAI routes @@ -1724,6 +1740,8 @@ class PromptTokensDetailsWrapper( image_tokens: int | None = None """Image tokens sent to the model.""" + cached_tokens_details: CachedTokensDetails | None = None + video_tokens: int | None = None """Video tokens sent to the model.""" @@ -2708,18 +2726,26 @@ class TranscriptionUsageTokensObject(BaseModel): input_tokens: int output_tokens: int total_tokens: int - input_token_details: TranscriptionUsageInputTokenDetailsObject + input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None + + +class TranscriptionDetectedLanguage(BaseModel): + code: str class TranscriptionResponse(OpenAIObject): text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None + languages: Sequence[TranscriptionDetectedLanguage] | None = None _hidden_params: dict = {} _response_headers: dict | None = None - def __init__(self, text=None) -> None: - super().__init__(text=text) + def __init__(self, text=None, usage=None, languages=None, **kwargs) -> None: # noqa: ANN003 # OpenAI-compatible response accepts provider extension fields + super().__init__(text=text, usage=usage, languages=languages, **kwargs) + + def set_audio_transcription_duration(self, duration: float) -> None: + self._hidden_params["audio_transcription_duration"] = duration def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/litellm/utils.py b/litellm/utils.py index dd35c17809f..d39d045468e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1242,6 +1242,8 @@ def function_setup( applied_guardrails=applied_guardrails, supports_correlation_logging=is_async_call, ) + if logging_obj is None: + raise RuntimeError("LiteLLM logging initialization returned no logger") ## check if metadata is passed in litellm_params: Final[dict[str, object]] = {"api_base": ""} @@ -1760,6 +1762,12 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.transcription.value and isinstance(result, openai.Stream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + wrap_transcription_stream, + ) + + result = wrap_transcription_stream(result, logging_obj, start_time) # RETURN RESULT update_response_metadata = getattr(sys.modules[__name__], "update_response_metadata") update_response_metadata( @@ -2062,6 +2070,12 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.atranscription.value and isinstance(result, openai.AsyncStream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + wrap_transcription_stream, + ) + + result = wrap_transcription_stream(result, logging_obj, start_time) _update_response_metadata( result=result, logging_obj=logging_obj, @@ -3441,10 +3455,13 @@ def get_optional_params_transcription( model: str, custom_llm_provider: str, language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: str | None = None, temperature: int | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, + stream: bool | None = None, drop_params: bool | None = None, **kwargs, ): @@ -3454,6 +3471,7 @@ def get_optional_params_transcription( passed_params: Final = locals() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") + passed_params.pop("model") custom_llm_provider = passed_params.pop("custom_llm_provider") drop_params = normalize_drop_params(passed_params.pop("drop_params")) special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs") @@ -3462,10 +3480,13 @@ def get_optional_params_transcription( default_params: Final = { "language": None, + "languages": None, + "keywords": None, "prompt": None, "response_format": None, "temperature": None, # openai defaults this to 0 "timestamp_granularities": None, + "stream": None, } non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} @@ -3522,6 +3543,9 @@ def get_optional_params_transcription( openai_params=OPENAI_TRANSCRIPTION_PARAMS, additional_drop_params=kwargs.get("additional_drop_params", None), ) + extra_body: Final = optional_params.get("extra_body") + if isinstance(extra_body, dict) and not extra_body: + optional_params.pop("extra_body") return optional_params @@ -6083,6 +6107,7 @@ def _get_model_info_helper( ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), + cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None @@ -8903,7 +8928,13 @@ class ProviderConfigManager: return XAIAudioTranscriptionConfig() elif litellm.LlmProviders.OPENAI == provider: - if "gpt-4o" in model: + if model == "gpt-transcribe": + from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, + ) + + return OpenAIGPTTranscribeAudioTranscriptionConfig() + elif "gpt-4o" in model: return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 3e389acc2a9..df12b6feea0 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -72,6 +72,10 @@ - {id: llm.bedrock_native.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native invoke stream"} - {id: llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock invoke missing fields and invalid temperature"} - {id: llm.ocr.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: ocr, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.13 / LIT-4778", rationale: "OCR missing document rejected"} +- {id: llm.realtime.openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: translation, streaming: stream, assertions: [works], source: "realtime_endpoints/endpoints.py", rationale: "Dedicated translation client-secret, raw SDP, and WebSocket paths emit translated audio and transcript deltas"} +- {id: llm.realtime.azure_openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: translation, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA translation session emits translated audio and transcript deltas"} +- {id: llm.realtime.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/realtime/handler.py", rationale: "gpt-live-transcribe and gpt-realtime-whisper emit live transcript deltas"} +- {id: llm.realtime.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA live transcription emits transcript deltas"} - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} @@ -89,6 +93,8 @@ - {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"} - {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"} - {id: llm.audio_transcriptions.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.7 / LIT-4778", rationale: "Transcription empty file and missing model are rejected"} +- {id: llm.audio_transcriptions.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "gpt-transcribe streams typed transcript delta and done events"} +- {id: llm.audio_transcriptions.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure gpt-transcribe streams typed transcript delta and done events over the v1 API"} - {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"} - {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"} - {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index e009b02b69c..43226b637eb 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -84,6 +84,8 @@ LlmCapability = Literal[ "tool_search", "tool_search_history", "tool_use", + "transcription", + "translation", "vision", "web_search", "web_search_server_tool", @@ -153,14 +155,7 @@ class OtherCell(_Base): Cell = Annotated[ - LlmCell - | MgmtCell - | McpCell - | ReliabilityCell - | QuotaCell - | LoggingCell - | GuardrailCell - | OtherCell, + LlmCell | MgmtCell | McpCell | ReliabilityCell | QuotaCell | LoggingCell | GuardrailCell | OtherCell, Field(discriminator="module"), ] diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 7e6d4d24905..38b27895a35 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.realtime_streaming import ( client_sent_openai_beta_realtime_header, ) from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks @@ -602,6 +603,42 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): assert "audio" not in session +@pytest.mark.asyncio +async def test_translation_session_update_omits_session_type(): + client_ws = MagicMock() + client_ws.scope = {"headers": []} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": {"output": {"language": "fr"}}, + }, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + translation_session=True, + ) + + await streaming.client_ack_messages() + + sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) + assert "type" not in sent_to_backend["session"] + assert sent_to_backend["session"]["audio"]["output"]["language"] == "fr" + + def test_translate_event_to_beta_renames_delta_types(): ev = RealTimeStreaming._translate_event_to_beta( {"type": "response.output_audio.delta", "delta": "abc", "event_id": "e1"} @@ -1023,6 +1060,93 @@ async def test_transcription_session_update_enforces_authorized_nested_model(): assert streaming._is_transcription_session is True +@pytest.mark.asyncio +async def test_translation_session_update_rejects_disallowed_nested_transcription_model() -> None: + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + translation_session=True, + ) + + with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe"}, + } + }, + }, + } + ) + ) + backend_ws.send.assert_not_awaited() + assert streaming._is_transcription_session is False + + +@pytest.mark.asyncio +async def test_translation_session_update_binds_nested_transcription_model() -> None: + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + translation_session=True, + ) + + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-realtime-whisper", "language": "en"}, + } + }, + }, + } + ) + ) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe", "language": "fr"}, + } + }, + }, + } + ) + ) + + first_sent = json.loads(backend_ws.send.await_args_list[0].args[0]) + second_sent = json.loads(backend_ws.send.await_args_list[1].args[0]) + assert first_sent["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + assert second_sent["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", + "language": "fr", + } + assert streaming._is_transcription_session is False + + @pytest.mark.asyncio async def test_normal_realtime_session_keeps_nested_transcription_model(): backend_ws = MagicMock() @@ -2786,6 +2910,69 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] +def test_translation_audio_duration_is_finalized_once(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + payload = base64.b64encode(bytes(48000)).decode() + streaming._capture_translation_output_audio({"type": "session.output_audio.delta", "delta": payload}) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +def test_translation_audio_duration_uses_session_output_format(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"output": {"format": {"type": "audio/pcmu", "rate": 8000}}}}, + } + ) + streaming._capture_translation_output_audio( + { + "type": "session.output_audio.delta", + "delta": base64.b64encode(bytes(8000)).decode(), + } + ) + streaming._finalize_translation_usage() + + assert streaming.messages[-1]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +def test_translation_does_not_duplicate_provider_duration_usage(): + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + streaming.messages.append({"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}}) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index caf941ebd19..26f66d4055c 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -474,6 +474,11 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "avector_store_search", "acreate_skill", "acreate_interaction", + "acreate_realtime_client_secret", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_calls", + "arealtime_translation_calls", ] ], ) diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index f7a88b5ba63..ebf93df7191 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,16 +1,31 @@ -import json from unittest.mock import AsyncMock, MagicMock, patch -import httpx import pytest - -from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context +from openai import AsyncOpenAI, omit +class DummySDKConnectionManager: + def __init__(self, connection): + self.connection = connection -@pytest.mark.parametrize( - "api_base", ["https://api.openai.com/v1", "https://api.openai.com"] -) + async def __aenter__(self): + return self.connection + + async def __aexit__(self, exc_type, exc, tb): + return None + + +def make_realtime_sdk_client(): + connection = MagicMock() + connection.send_raw = AsyncMock() + connection.recv_bytes = AsyncMock() + connection.close = AsyncMock() + client = MagicMock(spec=AsyncOpenAI) + client.realtime.connect = MagicMock(return_value=DummySDKConnectionManager(connection)) + return client + + +@pytest.mark.parametrize("api_base", ["https://api.openai.com/v1", "https://api.openai.com"]) def test_openai_realtime_handler_url_construction(api_base): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -59,12 +74,8 @@ def test_openai_realtime_handler_model_parameter_inclusion(): api_base = "https://api.openai.com/" # Test with just model parameter - query_params_model_only: RealtimeQueryParams = { - "model": "gpt-4o-mini-realtime-preview" - } - url = handler._construct_url( - api_base=api_base, query_params=query_params_model_only - ) + query_params_model_only: RealtimeQueryParams = {"model": "gpt-4o-mini-realtime-preview"} + url = handler._construct_url(api_base=api_base, query_params=query_params_model_only) # Verify the URL structure assert url.startswith("wss://api.openai.com/v1/realtime?") @@ -75,9 +86,7 @@ def test_openai_realtime_handler_model_parameter_inclusion(): "model": "gpt-4o-mini-realtime-preview", "intent": "chat", } - url_with_extras = handler._construct_url( - api_base=api_base, query_params=query_params_with_extras - ) + url_with_extras = handler._construct_url(api_base=api_base, query_params=query_params_with_extras) # Verify both parameters are included assert url_with_extras.startswith("wss://api.openai.com/v1/realtime?") @@ -91,11 +100,6 @@ def test_openai_realtime_handler_model_parameter_inclusion(): assert expected_pattern in url_with_extras -import asyncio - -import pytest - - @pytest.mark.asyncio async def test_async_realtime_success(): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -109,27 +113,10 @@ async def test_async_realtime_success(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -141,6 +128,7 @@ async def test_async_realtime_success(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) mock_realtime_streaming.assert_called_once() @@ -164,28 +152,10 @@ async def test_async_realtime_url_contains_model(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -197,30 +167,48 @@ async def test_async_realtime_url_contains_model(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called with the correct URL - mock_ws_connect.assert_called_once() - called_url = mock_ws_connect.call_args[0][0] - - # Verify the URL contains the model parameter - assert called_url.startswith("wss://api.openai.com/v1/realtime?") - assert f"model={model}" in called_url - - # Verify proper headers were set (GA default: no OpenAI-Beta unless client sent it) - called_kwargs = mock_ws_connect.call_args[1] - assert "additional_headers" in called_kwargs - additional_headers = called_kwargs["additional_headers"] + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] == model + additional_headers = called_kwargs["extra_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert "OpenAI-Beta" not in additional_headers - # Verify SSL is configured (should be an SSLContext or True, not None or False) - assert called_kwargs["ssl"] is not None - assert called_kwargs["ssl"] is not False + assert called_kwargs["max_retries"] == 0 mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() +@pytest.mark.asyncio +async def test_async_realtime_transcription_omits_sdk_model_query(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + websocket = AsyncMock() + logging_obj = MagicMock() + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: + mock_realtime_streaming.return_value.bidirectional_forward = AsyncMock() + + await handler.async_realtime( + model="gpt-live-transcribe", + websocket=websocket, + logging_obj=logging_obj, + api_key="test-key", + query_params={"intent": "transcription"}, + client=sdk_client, + ) + + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] is omit + assert called_kwargs["extra_query"] == {"intent": "transcription"} + + @pytest.mark.asyncio async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it(): """Upstream WS gets OpenAI-Beta: realtime=v1 only when the client WebSocket included it.""" @@ -240,26 +228,10 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() ] } dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -271,11 +243,12 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - mock_ws_connect.assert_called_once() - called_kwargs = mock_ws_connect.call_args[1] - additional_headers = called_kwargs["additional_headers"] + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + additional_headers = called_kwargs["extra_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert additional_headers["OpenAI-Beta"] == "realtime=v1" @@ -300,28 +273,10 @@ async def test_async_realtime_uses_max_size_parameter(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -333,20 +288,14 @@ async def test_async_realtime_uses_max_size_parameter(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called with the max_size parameter - mock_ws_connect.assert_called_once() - called_kwargs = mock_ws_connect.call_args[1] - - # Verify max_size is set (default None for unlimited, matching OpenAI's SDK) - assert "max_size" in called_kwargs - assert called_kwargs["max_size"] is None - # Verify SSL is configured (should be an SSLContext or True, not None or False) - assert called_kwargs["ssl"] is not None - assert called_kwargs["ssl"] is not False - # Default should be None (unlimited) to match OpenAI's official agents SDK - # https://github.com/openai/openai-agents-python/blob/cf1b933660e44fd37b4350c41febab8221801409/src/agents/realtime/openai_realtime.py#L235 + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + connection_options = called_kwargs["websocket_connection_options"] + assert connection_options["max_size"] is REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES + assert "ssl" not in connection_options mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() @@ -371,27 +320,10 @@ async def test_async_realtime_ws_url_has_no_ssl(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -403,19 +335,13 @@ async def test_async_realtime_ws_url_has_no_ssl(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called - mock_ws_connect.assert_called_once() - called_url = mock_ws_connect.call_args[0][0] - called_kwargs = mock_ws_connect.call_args[1] - - # Verify URL was converted from http:// to ws:// - assert called_url.startswith("ws://localhost:8113/v1/realtime?") - assert f"model={model}" in called_url - - # Verify ssl is None for ws:// URLs (the fix for issue #19222) - assert called_kwargs["ssl"] is None + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] == model + assert "ssl" not in called_kwargs["websocket_connection_options"] @pytest.mark.asyncio @@ -465,3 +391,65 @@ async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_ assert event["error"]["type"] == "server_error" assert "401" in event["error"]["message"] assert closed and closed[0][0] == 1008 + + +def test_translation_url_uses_dedicated_path(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + url = handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + assert url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + + +@pytest.mark.asyncio +async def test_translation_websocket_uses_direct_transport(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + backend = AsyncMock() + + class TranslationConnectionManager: + async def __aenter__(self): + return backend + + async def __aexit__(self, exc_type, exc, tb): + return None + + websocket = MagicMock() + websocket.scope = {"headers": []} + websocket.close = AsyncMock() + logging_obj = MagicMock() + handler = OpenAIRealtime() + expected_url = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert ( + handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + == expected_url + ) + + with ( + patch("websockets.connect", return_value=TranslationConnectionManager()) as connect, + patch( # test-quality-ok: transport test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as streaming, + ): + streaming.return_value.bidirectional_forward = AsyncMock() + await handler.async_realtime( + model="gpt-realtime-translate", + websocket=websocket, + logging_obj=logging_obj, + api_base="https://api.openai.com/v1", + api_key="sk-test", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + + connect.assert_called_once() + assert connect.call_args.args[0] == expected_url + assert streaming.call_args.kwargs["translation_session"] is True diff --git a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py index 54f206d098d..c663efa341b 100644 --- a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py @@ -21,9 +21,7 @@ from litellm.types.realtime import RealtimeTranscriptionSessionRequest def test_openai_transcription_session_url(): cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -32,9 +30,7 @@ def test_openai_transcription_session_url_strips_trailing_v1(): """A /v1 suffix must not be duplicated in the path.""" cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com/v1", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com/v1", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -46,9 +42,18 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version(): model="whisper-deploy", api_version="2025-04-01-preview", ) - assert ( - url - == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + assert url == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + + +@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) +def test_azure_ga_realtime_http_urls(api_version): + cfg = AzureRealtimeHTTPConfig() + base = "https://my.openai.azure.com" + + assert cfg.get_complete_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/calls") + assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == ( + f"{base}/openai/v1/realtime/transcription_sessions" ) @@ -141,6 +146,30 @@ async def test_client_secret_handler_still_targets_client_secrets_url(): assert kwargs["url"] == "https://api.openai.com/v1/realtime/client_secrets" +@pytest.mark.asyncio +async def test_azure_client_secret_prefers_provider_qualified_routing_model(): + import litellm + + mock_response = MagicMock(spec=httpx.Response) + mock_client = MagicMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=mock_response) + + result = await litellm.acreate_realtime_client_secret( + model="azure/gpt-realtime-2.1", + session={"type": "realtime", "model": "gpt-realtime-2.1"}, + api_base="https://my.openai.azure.com", + api_key="azure-test-key", + api_version="v1", + client=mock_client, + ) + + assert result is mock_response + request = mock_client.post.call_args.kwargs + assert request["url"] == "https://my.openai.azure.com/openai/v1/realtime/client_secrets" + assert request["headers"]["api-key"] == "azure-test-key" + assert request["json"]["session"]["model"] == "gpt-realtime-2.1" + + @pytest.mark.asyncio async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): """ @@ -169,18 +198,14 @@ async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): assert kwargs["url"].endswith("/v1/realtime/transcription_sessions") # The litellm-only routing hint must not be forwarded upstream. assert "model" not in kwargs["json"] - assert kwargs["json"]["input_audio_transcription"] == { - "model": "gpt-realtime-whisper" - } + assert kwargs["json"]["input_audio_transcription"] == {"model": "gpt-realtime-whisper"} def test_append_query_params_skips_existing_keys(): from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler url = "wss://example.com/v1/realtime?model=gpt-4o" - result = BaseLLMHTTPHandler._append_query_params( - url, {"model": "ignored", "intent": "transcription"} - ) + result = BaseLLMHTTPHandler._append_query_params(url, {"model": "ignored", "intent": "transcription"}) assert "model=ignored" not in result assert "intent=transcription" in result diff --git a/tests/test_litellm/llms/openai/realtime/test_translation.py b/tests/test_litellm/llms/openai/realtime/test_translation.py new file mode 100644 index 00000000000..6eb2a618b93 --- /dev/null +++ b/tests/test_litellm/llms/openai/realtime/test_translation.py @@ -0,0 +1,265 @@ +import gzip +import json +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncOpenAI + +from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig +from litellm.types.realtime import RealtimeSessionConfig + + +def test_realtime_session_config_supports_translation_and_live_transcription_fields(): + session = RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio={ + "input": { + "transcription": { + "model": "gpt-live-transcribe", + "delay": "minimal", + "languages": ["en", "fr"], + "keywords": ["LiteLLM"], + } + }, + "output": {"language": "es"}, + }, + ) + + assert session.audio is not None + assert session.audio.input is not None + assert session.audio.input.transcription is not None + assert session.audio.input.transcription.delay == "minimal" + assert session.audio.input.transcription.languages == ["en", "fr"] + assert session.audio.input.transcription.keywords == ["LiteLLM"] + assert session.audio.output is not None + assert session.audio.output.language == "es" + + +@pytest.mark.parametrize( + "api_base,expected", + [ + ( + "https://api.openai.com", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ( + "https://api.openai.com/v1", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ], +) +def test_openai_translation_client_secret_url(api_base: str, expected: str): + config = OpenAIRealtimeHTTPConfig() + assert config.get_translation_client_secret_url(api_base, "gpt-realtime-translate") == expected + + +def test_openai_translation_calls_url(): + config = OpenAIRealtimeHTTPConfig() + assert ( + config.get_translation_calls_url("https://api.openai.com/v1", "gpt-realtime-translate") + == "https://api.openai.com/v1/realtime/translations/calls" + ) + + +def test_azure_translation_urls_use_ga_paths(): + config = AzureRealtimeHTTPConfig() + assert ( + config.get_translation_client_secret_url("https://example.openai.azure.com", "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets" + ) + assert ( + config.get_translation_calls_url("https://example.openai.azure.com", "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/calls" + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_custom_translation_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/client_secrets"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + request_data = {"session": {"type": "translation", "model": "gpt-realtime-translate"}} + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://api.openai.com", + api_key="sk-test", + request_data=request_data, + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + ) + + assert response.status_code == 200 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/client_secrets" + assert call["json"] == request_data + + +@pytest.mark.asyncio +async def test_azure_translation_client_secret_supports_entra_bearer_auth(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request( + "POST", + "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets", + ), + ) + ) + handler = BaseLLMHTTPHandler() + + await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.openai.azure.com", + api_key="", + request_data={"session": {"type": "translation", "model": "translate-deployment"}}, + logging_obj=MagicMock(), + timeout=10, + provider_config=AzureRealtimeHTTPConfig(), + model="translate-deployment", + extra_headers={"Authorization": "Bearer entra-token"}, + client=client, + ) + + headers = client.post.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer entra-token" + assert "api-key" not in headers + + +@pytest.mark.asyncio +async def test_translation_calls_use_translation_session_and_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 201, + content=b"v=0\r\n", + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/calls"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://api.openai.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + translation=True, + ) + + assert response.status_code == 201 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/calls" + assert call["headers"]["Content-Type"] == "application/sdp" + assert call["content"] == "v=0\r\n" + + +@pytest.mark.asyncio +async def test_standard_client_secret_uses_openai_sdk_resource(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/client_secrets" + return httpx.Response( + 200, + content=gzip.compress(b'{"value":"ek_test"}'), + headers={"content-encoding": "gzip"}, + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + logging_obj = MagicMock() + + response = await handler.async_realtime_client_secret_handler( + api_base="https://example.com", + api_key="sk-test", + request_data={"session": {"type": "realtime", "model": "gpt-realtime-2.1"}}, + logging_obj=logging_obj, + timeout=10, + client=openai_client, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 200 + assert response.json() == {"value": "ek_test"} + assert "content-encoding" not in response.headers + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_openai_sdk_custom_post(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/translations/client_secrets" + assert json.loads(request.content)["session"]["audio"]["output"]["language"] == "es" + return httpx.Response(200, json={"value": "ek_translation"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.com", + api_key="sk-test", + request_data={ + "session": { + "model": "gpt-realtime-translate", + "audio": {"output": {"language": "es"}}, + } + }, + logging_obj=MagicMock(), + timeout=10, + client=openai_client, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 200 + assert response.json() == {"value": "ek_translation"} + + +@pytest.mark.asyncio +async def test_translation_calls_use_openai_sdk_custom_post(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/translations/calls" + body = await request.aread() + assert request.headers["content-type"] == "application/sdp" + assert body == b"v=0\r\n" + return httpx.Response(201, content=b"v=0\r\n") + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="ek_test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://example.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=MagicMock(), + timeout=10, + model="gpt-realtime-translate", + client=openai_client, + translation=True, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 201 + assert response.text == "v=0\r\n" diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index e5179387f82..1780bb5a101 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -693,6 +693,37 @@ def test_realtime_webrtc_http_routes_classified_as_llm_api(route): assert RouteChecks.is_management_route(route=route) is False +@pytest.mark.parametrize( + "route", + [ + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", + ], +) +def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route): + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["openai_routes"], + ) + + assert RouteChecks.is_llm_api_route(route=route) is True + assert RouteChecks.is_management_route(route=route) is False + assert ( + RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=valid_token, + ) + is True + ) + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index de76c7257cf..a756454405e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -119,9 +119,7 @@ def patched_transcription(monkeypatch): return data monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - monkeypatch.setattr( - proxy_server, "check_file_size_under_limit", lambda **kwargs: True - ) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", lambda **kwargs: True) async def _form_data(request): from starlette.datastructures import FormData, UploadFile @@ -153,6 +151,47 @@ def patched_transcription_error(monkeypatch, patched_transcription): yield +@pytest.fixture +def patched_transcription_stream(monkeypatch, patched_transcription): + class _FakeEvent: + def model_dump_json(self): + return '{"type":"transcript.text.done","text":"hello world"}' + + class _FakeAsyncStream: + def __init__(self): + self.closed = False + + def __aiter__(self): + async def _events(): + yield _FakeEvent() + + return _events() + + async def aclose(self): + self.closed = True + + async def _form_data(request): + from starlette.datastructures import FormData, UploadFile + + upload = UploadFile( + filename="audio.mp3", + file=io.BytesIO(b"\x00\x01\x02"), + ) + return FormData([("file", upload), ("model", "gpt-transcribe"), ("stream", "true")]) + + stream = _FakeAsyncStream() + + async def _llm_call(): + return stream + + async def _fake_route_request(*args, **kwargs): + return _llm_call() + + monkeypatch.setattr(proxy_server, "get_form_data", _form_data) + monkeypatch.setattr(proxy_server, "route_request", _fake_route_request) + yield stream + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_happy_path(client, auth_as, patched_speech, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (happy).""" @@ -254,3 +293,15 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error, response = client.post(path, files=files, data=data) assert response.status_code == 500 assert len(response.content) > 0 + + +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcription_stream, path): + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + data = {"model": "gpt-transcribe", "stream": "true"} + with auth_as(): + response = client.post(path, files=files, data=data) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\n' + assert patched_transcription_stream.closed is True diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index f5c97142dde..10c2543021e 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -15,16 +15,25 @@ import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient - -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ConfigGeneralSettings, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) from litellm.proxy.realtime_endpoints.endpoints import ( + _ALLOWED_SESSION_TYPES, + _coerce_realtime_session_type, _decode_realtime_token_payload, _encode_realtime_token_payload, + _prepare_client_secret_session, +) +from litellm.types.realtime import ( + RealtimeAudioInputConfig, + RealtimeAudioTranscriptionConfig, + RealtimeClientSecretRequest, + RealtimeSessionAudioConfig, + RealtimeSessionConfig, ) # --- Unit tests: token encode/decode helpers --- @@ -117,18 +126,107 @@ def proxy_app(monkeypatch): from litellm.proxy import proxy_server monkeypatch.setattr(proxy_server, "master_key", "sk-test-master-key") + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", True) return proxy_server.app +def test_non_billable_realtime_protocols_default_to_disabled(): + assert ConfigGeneralSettings().allow_non_billable_realtime_protocols is False + + +@pytest.mark.parametrize( + ("path", "body"), + ( + ( + "/v1/realtime/client_secrets", + {"model": "gpt-realtime-2"}, + ), + ( + "/v1/realtime/translations/client_secrets", + { + "model": "gpt-realtime-translate", + "session": {"type": "translation", "model": "gpt-realtime-translate"}, + }, + ), + ( + "/v1/realtime/transcription_sessions", + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + ), + ), +) +def test_non_billable_realtime_credential_endpoints_require_opt_in( + proxy_app, + monkeypatch, + path, + body, +): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") + try: + client = TestClient(proxy_app, raise_server_exceptions=False) + with patch( # test-quality-ok: endpoint gate must prove routing is never reached + "litellm.proxy.proxy_server.route_request" + ) as mock_route_request: + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json=body, + ) + + assert response.status_code == 403 + assert "bypasses LiteLLM billing" in response.json()["detail"] + mock_route_request.assert_not_called() + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + ( + "/v1/realtime/calls", + "/v1/realtime/translations/calls", + ), +) +def test_non_billable_realtime_sdp_endpoints_require_opt_in( + proxy_app, + monkeypatch, + path, +): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) + token_payload = _encode_realtime_token_payload( + ephemeral_key="epk", + model_id="gpt-realtime-2", + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + ) + encrypted_token = encrypt_value_helper(token_payload) + client = TestClient(proxy_app, raise_server_exceptions=False) + with patch( # test-quality-ok: endpoint gate must prove routing is never reached + "litellm.proxy.proxy_server.route_request" + ) as mock_route_request: + response = client.post( + path, + headers={"Authorization": f"Bearer {encrypted_token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 403 + assert "bypasses LiteLLM billing" in response.json()["detail"] + mock_route_request.assert_not_called() + + @pytest.fixture def mock_route_request_client_secrets(): """Mock route_request to return a fake upstream client_secrets response.""" future_expires_at = int(time.time()) + 3600 mock_resp = MagicMock(spec=httpx.Response) mock_resp.status_code = 200 - mock_resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) + mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode() mock_resp.headers = {} mock_resp.json.return_value = { @@ -215,9 +313,7 @@ async def test_client_secrets_success_with_mock( mock_pre_call_hook, ): """POST /v1/realtime/client_secrets returns 200 with valid auth and mocked upstream.""" - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -275,13 +371,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -312,12 +402,8 @@ async def test_client_secrets_transcription_routes_on_nested_model( async def _inner(): resp = MagicMock(spec=httpx.Response) resp.status_code = 200 - resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) - resp.content = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ).encode() + resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' + resp.content = (f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}').encode() resp.headers = {} resp.json.return_value = { "value": "upstream_ephemeral_key", @@ -351,13 +437,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -367,10 +447,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( session = captured["data"]["session"] assert session["type"] == "transcription" assert "model" not in session - assert ( - session["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert session["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" encrypted_value = response.json()["value"] decoded = _decode_realtime_token_payload( decrypt_value_helper( @@ -530,10 +607,7 @@ async def test_realtime_calls_replays_transcription_session_type( ) assert captured["session"]["type"] == "transcription" - assert ( - captured["session"]["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert captured["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" # --- transcription_sessions endpoint --- @@ -605,9 +679,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -651,9 +723,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -696,9 +766,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -751,9 +819,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -786,6 +852,20 @@ async def test_realtime_transcription_websocket_default_model_checks_key_scope() assert "is not available for this API key" in close_kwargs["reason"] +def test_realtime_transcription_upstream_query_omits_model(): + from litellm.proxy import proxy_server + + assert ( + proxy_server._resolve_realtime_upstream_query_model( + model="gpt-live-transcribe", + intent="transcription", + is_translation=False, + route_model="gpt-live-transcribe", + ) + is None + ) + + @pytest.mark.asyncio async def test_realtime_transcription_websocket_default_model_checks_team_scope(): from litellm.proxy import proxy_server @@ -947,9 +1027,7 @@ async def test_transcription_sessions_encrypts_client_secret( POST /v1/realtime/transcription_sessions returns 200 and the ephemeral key under client_secret.value must be encrypted (never the raw upstream key). """ - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") captured_route_type = {} async def _capturing_route(*args, **kwargs): @@ -993,16 +1071,12 @@ async def test_transcription_sessions_encrypts_client_secret( assert decrypted is not None assert "upstream_ephemeral_key" in decrypted # Routed through the dedicated transcription_sessions route type. - assert ( - captured_route_type["route_type"] - == "acreate_realtime_transcription_session" - ) + assert captured_route_type["route_type"] == "acreate_realtime_transcription_session" finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) def test_session_type_coerced_for_unknown_value(): - """An unrecognized session_type in the token falls back to 'realtime'.""" payload = _encode_realtime_token_payload( ephemeral_key="epk", model_id="gpt-4o", @@ -1011,12 +1085,222 @@ def test_session_type_coerced_for_unknown_value(): expires_at=None, session_type="INJECTED_TYPE", ) - # Force-deserialize and check the coercion that happens in proxy_realtime_calls. decoded = json.loads(payload) - session_type = decoded.get("session_type") or "realtime" - if session_type not in ("realtime", "transcription"): - session_type = "realtime" - assert session_type == "realtime" + assert decoded["session_type"] == "INJECTED_TYPE" + assert _coerce_realtime_session_type("INJECTED_TYPE") == "realtime" + assert _coerce_realtime_session_type(None) == "realtime" + for allowed_session_type in _ALLOWED_SESSION_TYPES: + assert _coerce_realtime_session_type(allowed_session_type) == allowed_session_type + + +@pytest.mark.asyncio +async def test_translation_client_secret_rejects_disallowed_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-live-transcribe"), + ) + ), + ), + ) + + with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_binds_authorized_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-realtime-whisper"), + ) + ), + ), + ) + + model, session_data, session_type = await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + assert model == "gpt-realtime-translate" + assert session_type == "translation" + assert session_data is not None + assert session_data["model"] == "gpt-realtime-translate" + assert session_data["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/client_secrets", + "/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + ], +) +def test_translation_client_secret_aliases_bind_token_family( + proxy_app, + mock_route_request_client_secrets, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-realtime-translate"], + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_client_secrets(*args, **kwargs) + + try: + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-realtime-translate"}, + ) + + assert response.status_code == 200 + assert captured["route_type"] == "acreate_realtime_translation_client_secret" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + decrypted = decrypt_value_helper( + response.json()["value"], + key="client_secret.value", + exception_type="debug", + ) + decoded = _decode_realtime_token_payload(decrypted or "") + assert decoded is not None + assert decoded["session_type"] == "translation" + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/calls", + "/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", + ], +) +def test_translation_calls_aliases_route_translation_session( + proxy_app, + mock_route_request_realtime_calls, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id="gpt-realtime-translate", + user_id="test-user", + team_id=None, + expires_at=int(time.time()) + 3600, + session_type="translation", + ) + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_realtime_calls(*args, **kwargs) + + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 201 + assert captured["route_type"] == "arealtime_translation_calls" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + + +@pytest.mark.parametrize( + "session_type,path", + [ + ("realtime", "/v1/realtime/translations/calls"), + ("translation", "/v1/realtime/calls"), + ], +) +def test_realtime_calls_reject_cross_family_token(proxy_app, session_type, path): + model = "gpt-realtime-translate" if session_type == "translation" else "gpt-realtime-2" + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id=model, + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + session_type=session_type, + ) + ) + response = TestClient(proxy_app).post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + assert response.status_code == 401 + assert response.json()["error"] == "Token is not valid for this Realtime endpoint" @pytest.mark.asyncio @@ -1142,9 +1426,7 @@ async def test_transcription_sessions_returns_upstream_error_verbatim( return _inner() - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -1184,9 +1466,7 @@ async def test_transcription_sessions_wraps_route_exception( async def _raise_http(*args, **kwargs): raise HTTPException(status_code=403, detail="Model not allowed") - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") try: client = TestClient(proxy_app, raise_server_exceptions=False) with ( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 89fd9c5c9d4..6f1e5dc9ef8 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -25,6 +25,7 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient +from starlette.datastructures import URL import litellm import litellm.proxy.proxy_server as proxy_server_module @@ -10757,7 +10758,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): def test_realtime_websocket_route_aliases_registered(): - """Realtime sessions reach the proxy via three path aliases stacked on + """Realtime sessions reach the proxy via six path aliases stacked on `realtime_websocket_endpoint`. Dropping any of them silently 405s WebSocket upgrades because the catch-all `/openai/{endpoint:path}` HTTP passthrough only declares HTTP methods. The aliases must also be @@ -10773,7 +10774,14 @@ def test_realtime_websocket_route_aliases_registered(): websocket_paths = {route.path for route in app.routes if isinstance(route, WebSocketRoute)} openai_routes = LiteLLMRoutes.openai_routes.value - for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + for expected in ( + "/openai/v1/realtime", + "/v1/realtime", + "/realtime", + "/openai/v1/realtime/translations", + "/v1/realtime/translations", + "/realtime/translations", + ): assert expected in websocket_paths, ( f"{expected!r} missing from registered WebSocket routes; the " f"realtime endpoint will 405 for clients hitting this path." @@ -10792,7 +10800,7 @@ def _lit6973_fake_realtime_ws() -> MagicMock: ws = MagicMock() ws.headers = {} ws.scope = {"headers": [], "type": "websocket"} - ws.url = "ws://testserver/v1/realtime" + ws.url = URL("ws://testserver/v1/realtime") ws.accept = AsyncMock() ws.send_text = AsyncMock() ws.close = AsyncMock() diff --git a/tests/test_litellm/responses/test_streaming_iterator_error_events.py b/tests/test_litellm/responses/test_streaming_iterator_error_events.py index e7cf09909fe..1d884e39682 100644 --- a/tests/test_litellm/responses/test_streaming_iterator_error_events.py +++ b/tests/test_litellm/responses/test_streaming_iterator_error_events.py @@ -538,6 +538,8 @@ def test_every_openai_sdk_response_error_code_has_explicit_status_mapping(): ("insufficient_quota", 429), ("vector_store_timeout", 504), ("invalid_prompt", 400), + ("data_residency_mismatch", 400), + ("bio_policy", 400), ("invalid_image", 400), ("invalid_image_format", 400), ("invalid_base64_image", 400), diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ce280cc3513..6428b0c96ee 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -979,6 +979,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/images/generations", "/v1/realtime", "/v1/realtime/transcription_sessions", + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls", "/v1/images/variations", "/v1/images/edits", "/v1/batch", diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py new file mode 100644 index 00000000000..6ccbeaf690a --- /dev/null +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -0,0 +1,297 @@ +import io +import json +import wave +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncOpenAI, AsyncStream, AzureOpenAI + +import litellm +from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, +) +from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription +from litellm.main import _validate_gpt_transcription_request +from litellm.types.utils import TranscriptionResponse +from litellm.utils import get_optional_params_transcription + + +def test_gpt_transcribe_config_uses_native_parameters_and_json(): + config = OpenAIGPTTranscribeAudioTranscriptionConfig() + supported = config.get_supported_openai_params("gpt-transcribe") + assert supported == ["prompt", "response_format", "keywords", "languages", "stream"] + + audio_file = io.BytesIO(b"audio") + request = config.transform_audio_transcription_request( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True}, + litellm_params={}, + ) + assert request.data["response_format"] == "json" + assert request.data["keywords"] == ["LiteLLM"] + assert request.data["languages"] == ["en", "fr"] + assert request.data["stream"] is True + + +def test_gpt_transcribe_optional_params_are_preserved(): + params = get_optional_params_transcription( + model="gpt-transcribe", + custom_llm_provider="openai", + keywords=["LiteLLM", "Realtime API"], + languages=["en", "fr"], + stream=True, + ) + assert params == { + "keywords": ["LiteLLM", "Realtime API"], + "languages": ["en", "fr"], + "stream": True, + } + + +def test_transcription_response_preserves_empty_languages(): + response = TranscriptionResponse(text="hello", languages=[]) + assert response.model_dump()["languages"] == [] + + +@pytest.mark.asyncio +async def test_openai_handler_returns_native_typed_stream(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + assert b'name="stream"' in body + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "languages": [], + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + handler = OpenAIAudioTranscription() + result = handler.audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"], "stream": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + logging_obj=logging_obj, + api_key="sk-test", + api_base="https://example.com/v1", + client=openai_client, + atranscription=True, + provider_config=OpenAIGPTTranscribeAudioTranscriptionConfig(), + ) + stream = await result + assert isinstance(stream, AsyncStream) + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now()) + received = [event async for event in wrapped_stream] + await wrapped_stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + assert received[-1].languages == [] + logging_obj.async_success_handler.assert_awaited_once() + logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + assert logged_response.text == "hello world" + assert logged_response.languages == [] + + +@pytest.mark.asyncio +async def test_atranscription_stream_preserves_duration_for_callback_cost(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + audio_file = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + + stream = await litellm.atranscription( + model="openai/gpt-transcribe", + file=audio_file, + stream=True, + client=openai_client, + litellm_logging_obj=logging_obj, + ) + received = [event async for event in stream] + await stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + logging_obj.async_success_handler.assert_awaited_once() + logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) + + +def test_gpt_transcribe_rejects_conflicting_language_inputs(): + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="cannot be used together"): + litellm.transcription( + model="gpt-transcribe", + file=audio_file, + language="en", + languages=["fr"], + api_key="sk-test", + ) + + +def test_gpt_transcribe_rejects_whisper_response_formats(): + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format='json'"): + litellm.transcription( + model="gpt-transcribe", + file=audio_file, + response_format="verbose_json", + api_key="sk-test", + ) + + +def test_gpt_live_transcribe_rejects_file_transcription(): + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="Realtime API"): + litellm.transcription( + model="gpt-live-transcribe", + file=audio_file, + api_key="sk-test", + ) + + +def test_azure_async_gpt_transcribe_forwards_v1_api_version(): + handler = AzureAudioTranscription() + handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) + + handler.audio_transcriptions( + model="gpt-transcribe", + audio_file=io.BytesIO(b"audio"), + optional_params={"stream": True}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="sk-test", + api_base="https://example.openai.azure.com", + api_version="v1", + atranscription=True, + ) + + assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == "v1" + + +@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) +def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: str | None): + resolved_api_version = _validate_gpt_transcription_request( + model="gpt-transcribe", + custom_llm_provider="azure", + language=None, + languages=None, + response_format="json", + api_version=api_version, + ) + + assert resolved_api_version == litellm.AZURE_DEFAULT_API_VERSION + + +def test_azure_gpt_transcribe_uses_deployment_scoped_route(): + def send_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ( + "https://example.openai.azure.com/openai/deployments/gpt-transcribe/audio/transcriptions" + f"?api-version={litellm.AZURE_DEFAULT_API_VERSION}" + ) + return httpx.Response( + 200, + json={"text": "hello", "languages": [{"code": "en"}], "usage": {"type": "duration", "seconds": 1}}, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(send_response)) + client = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert response.languages is not None + assert [language.code for language in response.languages] == ["en"] + client.close() + + +def test_azure_gpt_transcribe_preserves_dated_api_version(): + resolved_api_version = _validate_gpt_transcription_request( + model="gpt-transcribe", + custom_llm_provider="azure", + language=None, + languages=None, + response_format="json", + api_version="2025-04-01-preview", + ) + + assert resolved_api_version == "2025-04-01-preview" diff --git a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py index 703fa13cbc9..a15ba6d7543 100644 --- a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py +++ b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py @@ -9,6 +9,9 @@ TranscriptionVerbose/Diarized type. from unittest.mock import patch +import pytest + +import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, @@ -55,10 +58,7 @@ class TestDiarizedJsonUsageParsing: assert result.usage.seconds == 295.8 def test_usage_duration_object_accepts_float_seconds(self): - assert ( - TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds - == 295.8 - ) + assert TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds == 295.8 class TestTranscriptionDurationNotInResponseBody: @@ -126,6 +126,60 @@ class TestTranscriptionDurationNotInResponseBody: class TestCostCalculatorReadsDurationFromHiddenParams: """The cost calculator should read duration from _hidden_params via completion_cost().""" + @patch( # test-quality-ok: isolates duration selection from the model pricing table + "litellm.cost_calculator.openai_cost_per_second" + ) + def test_completion_cost_prefers_provider_usage_duration(self, mock_cost_fn): + mock_cost_fn.return_value = (0.001, 0.0) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = { + "audio_transcription_duration": 17.5, + "custom_llm_provider": "openai", + } + + completion_cost( + completion_response=response, + model="gpt-transcribe", + call_type="atranscription", + ) + + mock_cost_fn.assert_called_once() + _, kwargs = mock_cost_fn.call_args + assert kwargs["duration"] == 60.0 + + @pytest.mark.parametrize( + ("model", "provider"), + [ + ("gpt-transcribe", "openai"), + ("azure/gpt-transcribe", "azure"), + ], + ) + def test_gpt_transcribe_provider_usage_duration_costs_without_local_decode( + self, + monkeypatch, + model: str, + provider: str, + ): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = {"custom_llm_provider": provider} + + actual = completion_cost( + completion_response=response, + model=model, + call_type="atranscription", + custom_llm_provider=provider, + ) + + assert actual == pytest.approx(0.0045) + @patch("litellm.cost_calculator.openai_cost_per_second") def test_completion_cost_uses_hidden_params_duration(self, mock_cost_fn): """ From ac4946477b480e436ac97d4c9848734f325c9e04 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 12:40:44 -0500 Subject: [PATCH 02/21] fix(transcription): type SDK stream duration metadata explicitly --- litellm/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 376645f55f4..3d3a0bb7ec6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7842,7 +7842,7 @@ async def atranscription( if file is not None: calculated_duration = calculate_request_duration(file) if calculated_duration is not None: - setattr(response, "_litellm_audio_duration", calculated_duration) + response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field return response if not isinstance(response, TranscriptionResponse): raise ValueError( @@ -8187,7 +8187,7 @@ def transcription( calculated_duration: Final = calculate_request_duration(file) if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - setattr(response, "_litellm_audio_duration", calculated_duration) + response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field elif getattr(response, "duration", None) is None and calculated_duration is not None: response.set_audio_transcription_duration(calculated_duration) From a290cc5a7cef5f8cb7ce59c88250dfe7733363b2 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:03:12 -0500 Subject: [PATCH 03/21] fix(realtime): preserve transport settings and translation duration --- .../litellm_core_utils/realtime_streaming.py | 6 ++- litellm/llms/custom_httpx/llm_http_handler.py | 39 ++++++++++--------- litellm/llms/openai/realtime/handler.py | 7 ++-- .../test_realtime_streaming.py | 5 ++- .../realtime/test_openai_realtime_handler.py | 21 +++++++++- .../llms/openai/realtime/test_translation.py | 8 +++- 6 files changed, 58 insertions(+), 28 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 5e9e04ae54f..47ef8369e56 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -438,7 +438,11 @@ class RealTimeStreaming: if not self._is_translation_session: return self._capture_translation_output_format(event_obj) - if event_obj.get("type") != "session.output_audio.delta": + if event_obj.get("type") not in ( + "session.output_audio.delta", + "response.output_audio.delta", + "response.audio.delta", + ): return delta: Final = event_obj.get("delta") if not isinstance(delta, str): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 052b9ee9e87..c8f2bb1af57 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6469,17 +6469,7 @@ class BaseLLMHTTPHandler: cast_to=httpx.Response, body=request_data, ) - response_headers: Final = { # mutable-ok: httpx requires a concrete response-header mapping - key: value # mutable-ok: transport headers are materialized after filtering - for key, value in raw_response.headers.items() # mutable-ok: transport headers are materialized - if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") - } - return httpx.Response( - status_code=raw_response.status_code, - headers=response_headers, - content=raw_response.content, - request=httpx.Request("POST", f"{normalized_api_base}/realtime/client_secrets"), - ) + return self._decoded_realtime_sdk_response(raw_response) finally: if owns_client: await openai_client.close() @@ -6564,11 +6554,12 @@ class BaseLLMHTTPHandler: key: str(value) for key, value in (extra_headers or {}).items() }, ) - return await configured_client.post( + raw_response: Final = await configured_client.post( "/realtime/translations/client_secrets", cast_to=httpx.Response, body=request_data, ) + return self._decoded_realtime_sdk_response(raw_response) finally: if owns_client: await openai_client.close() @@ -6706,11 +6697,12 @@ class BaseLLMHTTPHandler: }, }, ) - return await configured_client.post( + translation_response: Final = await configured_client.post( "/realtime/translations/calls", cast_to=httpx.Response, content=sdp_text.encode("utf-8"), ) + return self._decoded_realtime_sdk_response(translation_response) realtime_session_data: Final = cast( # cast-ok: endpoint validation produced an OpenAI realtime session RealtimeSessionCreateRequestParam, session_data, @@ -6724,16 +6716,25 @@ class BaseLLMHTTPHandler: extra_headers=sdk_extra_headers, timeout=timeout, ) - return httpx.Response( - status_code=raw_response.status_code, - headers=raw_response.headers, - content=raw_response.content, - request=httpx.Request("POST", f"{normalized_api_base}/realtime/calls"), - ) + return self._decoded_realtime_sdk_response(raw_response.http_response) finally: if owns_client: await openai_client.close() + @staticmethod + def _decoded_realtime_sdk_response(response: httpx.Response) -> httpx.Response: + headers: Final = { # mutable-ok: httpx accepts a concrete response header mapping + key: value + for key, value in response.headers.items() + if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") + } + return httpx.Response( + status_code=response.status_code, + headers=headers, + content=response.content, + request=response.request, + ) + @staticmethod def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: if isinstance(client, AsyncHTTPHandler): diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index df2d9ca4c75..d3fc39f6f74 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -174,15 +174,14 @@ class OpenAIRealtime(OpenAIChatCompletion): ) -> AbstractAsyncContextManager[object]: import websockets - if realtime_mode == "translation" or client is None: + if realtime_mode == "translation" or not isinstance(client, AsyncOpenAI): return websockets.connect( url, additional_headers=headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=ssl_config, + **({"open_timeout": timeout} if timeout is not None else {}), ) - if not isinstance(client, AsyncOpenAI): - raise TypeError("client must be an AsyncOpenAI instance") openai_client: Final = client model_query: Final = query_params.get("model") extra_query: Final = { # mutable-ok: OpenAI SDK accepts a mutable query-parameter mapping @@ -195,6 +194,8 @@ class OpenAIRealtime(OpenAIChatCompletion): extra_headers=headers, websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + **({"ssl": ssl_config} if url.startswith("wss://") else {}), + **({"open_timeout": timeout} if timeout is not None else {}), }, max_retries=0, ) diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 38b27895a35..33493ffe6c1 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -2910,7 +2910,8 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] -def test_translation_audio_duration_is_finalized_once(): +@pytest.mark.parametrize("event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"]) +def test_translation_audio_duration_is_finalized_once(event_type: str): import base64 streaming = RealTimeStreaming( @@ -2921,7 +2922,7 @@ def test_translation_audio_duration_is_finalized_once(): translation_session=True, ) payload = base64.b64encode(bytes(48000)).decode() - streaming._capture_translation_output_audio({"type": "session.output_audio.delta", "delta": payload}) + streaming._capture_translation_output_audio({"type": event_type, "delta": payload}) streaming._finalize_translation_usage() streaming._finalize_translation_usage() diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index ebf93df7191..425298c3d9a 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -295,7 +295,7 @@ async def test_async_realtime_uses_max_size_parameter(): called_kwargs = sdk_client.realtime.connect.call_args.kwargs connection_options = called_kwargs["websocket_connection_options"] assert connection_options["max_size"] is REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES - assert "ssl" not in connection_options + assert connection_options["ssl"] is not None mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() @@ -453,3 +453,22 @@ async def test_translation_websocket_uses_direct_transport(): connect.assert_called_once() assert connect.call_args.args[0] == expected_url assert streaming.call_args.kwargs["translation_session"] is True + + +@pytest.mark.parametrize("sdk_client", [True, False]) +def test_connection_manager_preserves_transport_settings(sdk_client: bool): + import ssl + from litellm.llms.openai.realtime.handler import OpenAIRealtime + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + client = make_realtime_sdk_client() if sdk_client else MagicMock(spec=AsyncHTTPHandler) + ssl_config = ssl.create_default_context() + with patch("websockets.connect") as connect: + OpenAIRealtime()._create_connection_manager( + api_base="https://example.com", api_key="test", model="gpt-realtime-2.1", + query_params={"model": "gpt-realtime-2.1"}, headers={}, timeout=7.0, + realtime_mode="realtime", ssl_config=ssl_config, client=client, url="wss://example.com/v1/realtime", + ) + options = client.realtime.connect.call_args.kwargs["websocket_connection_options"] if sdk_client else connect.call_args.kwargs + assert options["ssl"] is ssl_config + assert options["open_timeout"] == 7.0 diff --git a/tests/test_litellm/llms/openai/realtime/test_translation.py b/tests/test_litellm/llms/openai/realtime/test_translation.py index 6eb2a618b93..a329200b2f7 100644 --- a/tests/test_litellm/llms/openai/realtime/test_translation.py +++ b/tests/test_litellm/llms/openai/realtime/test_translation.py @@ -209,7 +209,7 @@ async def test_translation_client_secret_uses_openai_sdk_custom_post(): async def send_response(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/realtime/translations/client_secrets" assert json.loads(request.content)["session"]["audio"]["output"]["language"] == "es" - return httpx.Response(200, json={"value": "ek_translation"}) + return httpx.Response(200, content=gzip.compress(b'{"value":"ek_translation"}'), headers={"content-encoding": "gzip"}) http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) @@ -233,6 +233,8 @@ async def test_translation_client_secret_uses_openai_sdk_custom_post(): assert response.status_code == 200 assert response.json() == {"value": "ek_translation"} + assert "content-encoding" not in response.headers + assert int(response.headers["content-length"]) == len(response.content) @pytest.mark.asyncio @@ -242,7 +244,7 @@ async def test_translation_calls_use_openai_sdk_custom_post(): body = await request.aread() assert request.headers["content-type"] == "application/sdp" assert body == b"v=0\r\n" - return httpx.Response(201, content=b"v=0\r\n") + return httpx.Response(201, content=gzip.compress(b"v=0\r\n"), headers={"content-encoding": "gzip"}) http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) openai_client = AsyncOpenAI(api_key="ek_test", base_url="https://example.com/v1", http_client=http_client) @@ -263,3 +265,5 @@ async def test_translation_calls_use_openai_sdk_custom_post(): assert response.status_code == 201 assert response.text == "v=0\r\n" + assert "content-encoding" not in response.headers + assert int(response.headers["content-length"]) == len(response.content) From caea679de0e8bbe68d3ec703eeb901e48bfed77a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:10:23 -0500 Subject: [PATCH 04/21] refactor(realtime): reuse SDK client initialization --- litellm/llms/custom_httpx/llm_http_handler.py | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index c8f2bb1af57..3437a6f3348 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6438,16 +6438,7 @@ class BaseLLMHTTPHandler: header auth when available; falls back to the legacy OpenAI-style defaults. """ if use_openai_sdk: - trimmed_api_base: Final = api_base.rstrip("/") - normalized_api_base: Final = ( - trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" - ) - owns_client: Final = not isinstance(client, AsyncOpenAI) - openai_client: Final = ( - client - if isinstance(client, AsyncOpenAI) - else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) - ) + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) logging_obj.pre_call( input=request_data, api_key="", @@ -6530,15 +6521,7 @@ class BaseLLMHTTPHandler: use_openai_sdk: bool = False, ) -> httpx.Response: if use_openai_sdk: - normalized_api_base = api_base.rstrip("/") - if not normalized_api_base.endswith("/v1"): - normalized_api_base = f"{normalized_api_base}/v1" - owns_client: Final = not isinstance(client, AsyncOpenAI) - openai_client: Final = ( - client - if isinstance(client, AsyncOpenAI) - else AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0) - ) + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) logging_obj.pre_call( input=request_data, api_key="", @@ -6669,14 +6652,8 @@ class BaseLLMHTTPHandler: client: object | None, translation: bool, ) -> httpx.Response: - normalized_api_base = api_base.rstrip("/") - if not normalized_api_base.endswith("/v1"): - normalized_api_base = f"{normalized_api_base}/v1" - owns_client: Final = not isinstance(client, AsyncOpenAI) - openai_client: Final = ( - client - if isinstance(client, AsyncOpenAI) - else AsyncOpenAI(api_key=openai_ephemeral_key, base_url=normalized_api_base, max_retries=0) + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client( + api_base, openai_ephemeral_key, client ) logging_obj.pre_call( input="realtime_sdp_offer", @@ -6721,6 +6698,14 @@ class BaseLLMHTTPHandler: if owns_client: await openai_client.close() + @staticmethod + def _realtime_sdk_client(api_base: str, api_key: str, client: object | None) -> tuple[AsyncOpenAI, bool, str]: + trimmed_api_base: Final = api_base.rstrip("/") + normalized_api_base: Final = trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" + if isinstance(client, AsyncOpenAI): + return client, False, normalized_api_base + return AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0), True, normalized_api_base + @staticmethod def _decoded_realtime_sdk_response(response: httpx.Response) -> httpx.Response: headers: Final = { # mutable-ok: httpx accepts a concrete response header mapping From 8783a3b3011c2904d147a656102ed32f08cd2829 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:31:02 -0500 Subject: [PATCH 05/21] fix(audio): bill finalized streams and authorize transcription models --- litellm/cost_calculator.py | 20 ++++++++++++++++++- litellm/litellm_core_utils/litellm_logging.py | 18 ++++++++++++++--- .../usage_object_transformation.py | 10 +++++++--- litellm/proxy/realtime_endpoints/endpoints.py | 7 +++++++ litellm/utils.py | 2 +- .../test_realtime_webrtc_endpoints.py | 17 ++++++++++++++-- .../transcriptions/test_gpt_transcribe.py | 15 ++++++++------ 7 files changed, 73 insertions(+), 16 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 2e4448475a7..82acd10c186 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -628,7 +628,25 @@ def cost_per_token( data_residency=data_residency, ) elif call_type == "atranscription" or call_type == "transcription": - if _transcription_usage_has_token_details(usage_block): + transcription_model_info: Final = _cached_get_model_info_helper( + model=model_without_prefix, custom_llm_provider=custom_llm_provider + ) + has_token_pricing: Final = any( + transcription_model_info.get(field) + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + ) + ) + if _transcription_usage_has_token_details(usage_block) and ( + has_token_pricing + or ( + transcription_model_info.get("input_cost_per_second") is None + and transcription_model_info.get("output_cost_per_second") is None + ) + ): return generic_cost_per_token( model=model_without_prefix, usage=usage_block, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8e28a0d543d..3eba410b4aa 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3286,7 +3286,12 @@ class Logging(LiteLLMLoggingBaseClass): if "async_complete_streaming_response" in self.model_call_details: return # break out of this. complete_streaming_response: Final[ - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None ] = self._get_assembled_streaming_response( result=result, start_time=start_time, @@ -4105,10 +4110,17 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: list[object], - ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None: + ) -> ( + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None + ): if self.stream is not True: return None - if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse): + if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)): return result elif isinstance( result, diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index a02c40b7611..a34e4de2dca 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -31,9 +31,13 @@ class TranscriptionUsageObjectTransformation: prompt_tokens=usage_object.input_tokens, completion_tokens=usage_object.output_tokens, total_tokens=usage_object.total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=usage_object.input_token_details.text_tokens, - audio_tokens=usage_object.input_token_details.audio_tokens, + prompt_tokens_details=( + PromptTokensDetailsWrapper( + text_tokens=usage_object.input_token_details.text_tokens, + audio_tokens=usage_object.input_token_details.audio_tokens, + ) + if usage_object.input_token_details is not None + else None ), ) return None diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index bee2a55f5b5..eb3a8430923 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -675,6 +675,13 @@ async def create_realtime_transcription_session( ) transcription_session: Final = {k: v for k, v in body.items() if k != "model"} + await _authorize_and_bind_nested_transcription_models( + session_data=transcription_session, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + _set_transcription_model_on_session(session=transcription_session, model=model, create_if_missing=True) data = {"model": model, "transcription_session": transcription_session} data = await add_litellm_data_to_request( diff --git a/litellm/utils.py b/litellm/utils.py index d39d045468e..ba00ae4c6d3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3468,7 +3468,7 @@ def get_optional_params_transcription( from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS # retrieve all parameters passed to the function - passed_params: Final = locals() + passed_params: Final = locals().copy() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") passed_params.pop("model") diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 10c2543021e..19242d7bdd2 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -661,8 +661,14 @@ def test_transcription_sessions_requires_auth(proxy_app): @pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + {"model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"input_audio_transcription": {"model": "gpt-4o-realtime-preview"}, "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "gpt-realtime-whisper"}}, +]) async def test_transcription_sessions_rejects_disallowed_resolved_model( - proxy_app, + proxy_app, body, ): proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( user_id="test-user", @@ -679,7 +685,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + json=body, ) assert response.status_code == 403 @@ -1032,6 +1038,7 @@ async def test_transcription_sessions_encrypts_client_secret( async def _capturing_route(*args, **kwargs): captured_route_type["route_type"] = kwargs.get("route_type") + captured_route_type["session"] = kwargs["data"]["transcription_session"] return await mock_route_request_transcription_sessions(*args, **kwargs) try: @@ -1054,6 +1061,8 @@ async def test_transcription_sessions_encrypts_client_secret( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, json={ + "model": "gpt-realtime-whisper", + "audio": {"input": {"transcription": {"model": "gpt-live-transcribe", "language": "en"}}}, "input_audio_format": "pcm16", "input_audio_transcription": {"model": "gpt-realtime-whisper"}, }, @@ -1072,6 +1081,10 @@ async def test_transcription_sessions_encrypts_client_secret( assert "upstream_ephemeral_key" in decrypted # Routed through the dedicated transcription_sessions route type. assert captured_route_type["route_type"] == "acreate_realtime_transcription_session" + assert captured_route_type["session"]["input_audio_transcription"]["model"] == "gpt-realtime-whisper" + assert captured_route_type["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", "language": "en", + } finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 6ccbeaf690a..35bed209400 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -143,10 +143,12 @@ async def test_atranscription_stream_preserves_duration_for_callback_cost(): http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) - logging_obj = MagicMock() - logging_obj.model_call_details = {} - logging_obj.async_success_handler = AsyncMock() - logging_obj.async_failure_handler = AsyncMock() + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-transcribe", messages=[], stream=True, call_type="atranscription", + start_time=datetime.now(), litellm_call_id="transcription-cost-test", function_id="transcription-cost-test", + ) audio_file = io.BytesIO() with wave.open(audio_file, "wb") as wav_file: wav_file.setnchannels(1) @@ -167,8 +169,9 @@ async def test_atranscription_stream_preserves_duration_for_callback_cost(): await openai_client.close() assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] - logging_obj.async_success_handler.assert_awaited_once() - logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + logged_response = logging_obj.model_call_details["async_complete_streaming_response"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.000075) + assert logging_obj.model_call_details["standard_logging_object"]["response_cost"] == pytest.approx(0.000075) assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) From ea84863217154370695c37e500ee2242d815bda6 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:35:42 -0500 Subject: [PATCH 06/21] refactor(audio): keep transcription pricing selection together --- litellm/cost_calculator.py | 39 +++++++++---------- litellm/litellm_core_utils/litellm_logging.py | 9 +---- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 82acd10c186..a256bc85425 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -296,12 +296,29 @@ def _get_additional_costs( return None -def _transcription_usage_has_token_details( +def _transcription_uses_token_pricing( + model: str, + custom_llm_provider: str | None, usage_block: Usage | None, ) -> bool: if usage_block is None: return False + model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + has_token_pricing: Final = any( + model_info.get(field) + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + ) + ) + if not has_token_pricing and ( + model_info.get("input_cost_per_second") is not None or model_info.get("output_cost_per_second") is not None + ): + return False + prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0 completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0 prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None) @@ -628,25 +645,7 @@ def cost_per_token( data_residency=data_residency, ) elif call_type == "atranscription" or call_type == "transcription": - transcription_model_info: Final = _cached_get_model_info_helper( - model=model_without_prefix, custom_llm_provider=custom_llm_provider - ) - has_token_pricing: Final = any( - transcription_model_info.get(field) - for field in ( - "input_cost_per_token", - "output_cost_per_token", - "input_cost_per_audio_token", - "output_cost_per_audio_token", - ) - ) - if _transcription_usage_has_token_details(usage_block) and ( - has_token_pricing - or ( - transcription_model_info.get("input_cost_per_second") is None - and transcription_model_info.get("output_cost_per_second") is None - ) - ): + if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block): return generic_cost_per_token( model=model_without_prefix, usage=usage_block, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3eba410b4aa..f1ac550f182 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3285,14 +3285,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE if "async_complete_streaming_response" in self.model_call_details: return # break out of this. - complete_streaming_response: Final[ - ModelResponse - | TextCompletionResponse - | ResponsesAPIResponse - | InteractionsAPIResponse - | TranscriptionResponse - | None - ] = self._get_assembled_streaming_response( + complete_streaming_response: Final = self._get_assembled_streaming_response( result=result, start_time=start_time, end_time=end_time, From 257c7f2e0a5215a0148d92ecc3245107f9b92af1 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 18:27:12 -0500 Subject: [PATCH 07/21] fix(realtime): reconcile model metadata and test contracts --- litellm/llms/custom_httpx/llm_http_handler.py | 18 +- litellm/llms/openai/realtime/handler.py | 12 +- ...odel_prices_and_context_window_backup.json | 275 ++++++++++++-- model_prices_and_context_window.json | 275 ++++++++++++-- .../llm_cost_calc/test_llm_cost_calc_utils.py | 48 ++- .../realtime/test_openai_realtime_handler.py | 1 + .../test_realtime_webrtc_endpoints.py | 2 +- tests/test_litellm/test_cost_calculator.py | 74 +++- .../test_litellm/test_model_prices_schema.py | 33 ++ .../realtime/test_azure_realtime_handler.py | 15 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 356 +++++++++++++++++- 11 files changed, 1012 insertions(+), 97 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 3437a6f3348..dc76142a14f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6533,9 +6533,9 @@ class BaseLLMHTTPHandler: try: configured_client: Final = openai_client.with_options( timeout=timeout, - set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping - key: str(value) for key, value in (extra_headers or {}).items() - }, + set_default_headers=MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ), ) raw_response: Final = await configured_client.post( "/realtime/translations/client_secrets", @@ -6669,9 +6669,9 @@ class BaseLLMHTTPHandler: timeout=timeout, set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping "Content-Type": "application/sdp", - **{ # mutable-ok: caller headers are normalized into the SDK header mapping - key: str(value) for key, value in (extra_headers or {}).items() - }, + **MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ), }, ) translation_response: Final = await configured_client.post( @@ -6684,9 +6684,9 @@ class BaseLLMHTTPHandler: RealtimeSessionCreateRequestParam, session_data, ) - sdk_extra_headers: Final = { # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping - key: str(value) for key, value in (extra_headers or {}).items() - } + sdk_extra_headers: Final = MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ) raw_response: Final = await openai_client.realtime.calls.with_raw_response.create( sdp=sdp_text, session=realtime_session_data, diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index d3fc39f6f74..c7801e8f201 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -7,7 +7,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. import ssl from collections.abc import Mapping from contextlib import AbstractAsyncContextManager -from types import TracebackType +from types import MappingProxyType, TracebackType from typing import Any, Final, cast from openai import AsyncOpenAI, omit @@ -180,13 +180,11 @@ class OpenAIRealtime(OpenAIChatCompletion): additional_headers=headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=ssl_config, - **({"open_timeout": timeout} if timeout is not None else {}), + **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), ) openai_client: Final = client model_query: Final = query_params.get("model") - extra_query: Final = { # mutable-ok: OpenAI SDK accepts a mutable query-parameter mapping - key: value for key, value in query_params.items() if key != "model" - } + extra_query: Final = MappingProxyType({key: value for key, value in query_params.items() if key != "model"}) sdk_model: Final = omit if query_params.get("intent") == "transcription" else model_query or model sdk_connection_manager: Final = openai_client.realtime.connect( model=sdk_model, @@ -194,8 +192,8 @@ class OpenAIRealtime(OpenAIChatCompletion): extra_headers=headers, websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - **({"ssl": ssl_config} if url.startswith("wss://") else {}), - **({"open_timeout": timeout} if timeout is not None else {}), + **(MappingProxyType({"ssl": ssl_config}) if url.startswith("wss://") else MappingProxyType({})), + **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), }, max_retries=0, ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ae02744b8e1..83525938f23 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,4 +1,229 @@ { + "azure/gpt-live-transcribe": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-transcribe": { + "input_cost_per_second": 7.5e-05, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/audio/transcriptions", + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -5821,6 +6046,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 3.2e-05, @@ -5850,12 +6076,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 1e-05, @@ -5885,6 +6114,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -25462,7 +25693,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -25555,7 +25785,6 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -25639,7 +25868,6 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -25732,7 +25960,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -25792,7 +26019,6 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -25849,7 +26075,6 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -26455,7 +26680,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -26515,7 +26739,6 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26573,7 +26796,6 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26632,7 +26854,6 @@ "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28380,7 +28601,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -28440,7 +28660,6 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28498,7 +28717,6 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28557,7 +28775,6 @@ "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -33245,7 +33462,6 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -33426,7 +33642,6 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -33527,7 +33742,6 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, - "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -33715,6 +33929,7 @@ "gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -33745,12 +33960,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -33781,6 +33999,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -47859,7 +48079,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -47904,7 +48123,6 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -47940,7 +48158,6 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -48033,7 +48250,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -48094,7 +48310,6 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -48152,7 +48367,6 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -57348,7 +57562,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -57366,7 +57581,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -57392,7 +57608,13 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], "supported_modalities": [ "audio" ], @@ -57401,7 +57623,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ae02744b8e1..83525938f23 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,4 +1,229 @@ { + "azure/gpt-live-transcribe": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-transcribe": { + "input_cost_per_second": 7.5e-05, + "litellm_provider": "azure", + "mode": "audio_transcription", + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/audio/transcriptions", + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -5821,6 +6046,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 3.2e-05, @@ -5850,12 +6076,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 1e-05, @@ -5885,6 +6114,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -25462,7 +25693,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -25555,7 +25785,6 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -25639,7 +25868,6 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -25732,7 +25960,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -25792,7 +26019,6 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -25849,7 +26075,6 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -26455,7 +26680,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -26515,7 +26739,6 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26573,7 +26796,6 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26632,7 +26854,6 @@ "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28380,7 +28601,6 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, - "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -28440,7 +28660,6 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28498,7 +28717,6 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28557,7 +28775,6 @@ "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, - "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -33245,7 +33462,6 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, - "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -33426,7 +33642,6 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -33527,7 +33742,6 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, - "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -33715,6 +33929,7 @@ "gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "input_cost_per_audio_token": 3.2e-05, "input_cost_per_image_token": 5e-06, @@ -33745,12 +33960,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "input_cost_per_audio_token": 1e-05, "input_cost_per_image_token": 8e-07, @@ -33781,6 +33999,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -47859,7 +48079,6 @@ "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -47904,7 +48123,6 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, - "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -47940,7 +48158,6 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -48033,7 +48250,6 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, - "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -48094,7 +48310,6 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -48152,7 +48367,6 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -57348,7 +57562,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -57366,7 +57581,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -57392,7 +57608,13 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], "supported_modalities": [ "audio" ], @@ -57401,7 +57623,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 76ccdec25d0..c87a416bb0f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -29,6 +29,7 @@ from litellm.llms.gemini.image_generation.cost_calculator import ( from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, @@ -42,6 +43,39 @@ from litellm.types.utils import ( ) +def test_realtime_cached_modality_breakdown_matches_prompt_cost(_local_model_cost_map): + model: Final = "gpt-realtime-2.1-mini" + rates: Final = litellm.model_cost[model] + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=400, + image_tokens=200, + cached_tokens=300, + cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=150, image_tokens=50), + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + breakdown: Final = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) + cached_cost: Final = ( + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + ) + uncached_cost: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + ) + + assert breakdown.cache_read_cost == pytest.approx(cached_cost) + assert prompt_cost == pytest.approx(uncached_cost + cached_cost) + + @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -299,8 +333,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ) - - def test_image_tokens_with_custom_pricing(): """Test that image_tokens in completion are properly costed with output_cost_per_image_token.""" from unittest.mock import patch @@ -1950,6 +1982,10 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_tokens_details: PromptTokensDetailsResult = { "cache_hit_tokens": 0, "cache_hit_audio_tokens": 0, + "cached_text_tokens": 0, + "cached_audio_tokens": 0, + "cached_image_tokens": 0, + "has_cached_tokens_details": False, "cache_creation_tokens": 0, "cache_creation_token_details": CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, @@ -2185,10 +2221,6 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo assert round(cost, 10) == round(expected_cost, 10) - - - - def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): usage = Usage( prompt_tokens=0, @@ -2367,8 +2399,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode assert base == located - - def test_vertex_uplift_invalid_multiplier_defaults_to_one(): """A malformed multiplier in the cost map degrades to base pricing, never raises.""" from litellm.litellm_core_utils.llm_cost_calc.utils import ( @@ -3585,8 +3615,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima assert cost == pytest.approx(0.07) - - @pytest.mark.parametrize( ("custom_llm_provider", "model"), [ diff --git a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py index 425298c3d9a..3d1c953492b 100644 --- a/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/test_litellm/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,3 +1,4 @@ +import json from unittest.mock import AsyncMock, MagicMock, patch import pytest diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index 19242d7bdd2..f9f75c39db4 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -1121,7 +1121,7 @@ async def test_translation_client_secret_rejects_disallowed_nested_transcription ), ) - with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available for this API key"): await _prepare_client_secret_session( req=req, user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index ddcdeb61e02..7362d36d523 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -160,10 +160,6 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 - - - - def test_realtime_stream_combines_text_and_audio_token_details(): """Realtime response.done usage with input_token_details / output_token_details.""" from litellm.cost_calculator import RealtimeAPITokenUsageProcessor @@ -587,7 +583,7 @@ def test_completion_cost_image_generation_reads_deployment_model_info_price_from assert cost == pytest.approx(0.08) -def test_completion_cost_image_generation_registered_deployment_price_keeps_map_token_rates( +def test_completion_cost_image_generation_registered_deployment_applies_custom_image_rate( _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: deployment_id: Final = "gemini-image-deployment-priced-per-image" @@ -1041,8 +1037,6 @@ def test_bedrock_cost_calculator_comparison_with_without_cache(): print(f"Cost with cache: {cost_with_cache}") - - def test_gemini_25_explicit_caching_cost_direct_usage(): """ Test that Gemini 2.5 models correctly calculate costs with explicit caching. @@ -1611,8 +1605,6 @@ def test_cost_margin_with_discount(monkeypatch): print(f" - Expected: ${expected_cost:.6f}") - - def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_map): """Test that completion_cost extracts service_tier from completion_response object.""" from litellm import completion_cost @@ -2363,8 +2355,6 @@ def test_gemini_without_cache_tokens_details(): print("✅ Gemini without cacheTokensDetails works correctly") - - def test_additional_costs_only_for_azure_ai(_local_model_cost_map): """ Test that _get_additional_costs is only called for azure_ai provider. @@ -4633,3 +4623,65 @@ def test_gemini_live_native_audio_limits_and_capabilities_match_vendor_model_car assert info["supports_response_schema"] is False assert info["supports_url_context"] is False assert info["supports_pdf_input"] is False + +@pytest.mark.parametrize("provider", ("openai", "azure")) +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str): + model_name: Final = f"azure/{model}" if provider == "azure" else model + rates: Final = litellm.model_cost[model_name] + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.created", "session": {"model": model}}, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 300, + "total_tokens": 1300, + "input_token_details": { + "text_tokens": 400, + "audio_tokens": 400, + "image_tokens": 200, + "cached_tokens": 300, + "cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50}, + }, + "output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100}, + } + }, + }, + ] + combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events) + actual: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=combined, + custom_llm_provider=provider, + litellm_model_name=model_name, + ) + expected: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + + 200 * rates["output_cost_per_token"] + + 100 * rates["output_cost_per_audio_token"] + ) + + assert actual == pytest.approx(expected) + + +def test_realtime_translation_duration_cost(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) diff --git a/tests/test_litellm/test_model_prices_schema.py b/tests/test_litellm/test_model_prices_schema.py index 052278631e2..24d533d9a3b 100644 --- a/tests/test_litellm/test_model_prices_schema.py +++ b/tests/test_litellm/test_model_prices_schema.py @@ -478,3 +478,36 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): "unknown_root-new_family_models", "vertex_ai-new_family_models", ] + + +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["cache_read_input_image_token_cost"] > 0 + assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"] + assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"] + + +@pytest.mark.parametrize( + "model,mode", + ( + ("gpt-realtime-translate", "realtime"), + ("gpt-live-transcribe", "audio_transcription"), + ("gpt-transcribe", "audio_transcription"), + ), +) +def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["mode"] == azure["mode"] == mode + assert azure["supports_audio_input"] is True + assert azure["supported_endpoints"] + + +def test_model_prices_backup_is_synchronized(prices: dict): + backup: Final = json.loads(BACKUP_PRICES_PATH.read_text()) + + assert backup == prices diff --git a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py index 73f43ec8d8a..de6deb4de5a 100644 --- a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py @@ -259,6 +259,21 @@ async def test_construct_url_v1_protocol(): assert url.count("/realtime") == 1 +def test_construct_url_translation_protocol(): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + + url = AzureOpenAIRealtime()._construct_url( + api_base="https://my-endpoint.openai.azure.com", + model="translate-deployment", + api_version=None, + realtime_protocol="GA", + query_params={"model": "translate-deployment"}, + realtime_mode="translation", + ) + + assert url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime/translations?model=translate-deployment" + + @pytest.mark.asyncio @pytest.mark.parametrize("protocol", ["ga", "Ga", "gA", "V1", "v1", "GA"]) async def test_construct_url_case_insensitive_protocol(protocol): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bdfd4aec316..b153e8a8cce 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10227,7 +10227,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_3"]; + get: operations["websocket_realtime_websocket_endpoint_get_6"]; put?: never; post?: never; delete?: never; @@ -10294,6 +10294,60 @@ export interface paths { patch?: never; trace?: never; }; + "/openai/v1/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get_3"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/openai/v1/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_openai_v1_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/openai/v1/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai/v1/responses": { parameters: { query?: never; @@ -13109,7 +13163,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get"]; + get: operations["websocket_realtime_websocket_endpoint_get_4"]; put?: never; post?: never; delete?: never; @@ -13176,6 +13230,60 @@ export interface paths { patch?: never; trace?: never; }; + "/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/register": { parameters: { query?: never; @@ -20205,7 +20313,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_2"]; + get: operations["websocket_realtime_websocket_endpoint_get_5"]; put?: never; post?: never; delete?: never; @@ -20272,6 +20380,60 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_v1_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_v1_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/rerank": { parameters: { query?: never; @@ -26043,7 +26205,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -27123,6 +27285,12 @@ export interface components { * @description opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine */ allow_cli_sso_verification_uri_complete?: boolean | null; + /** + * Allow Non Billable Realtime Protocols + * @description Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement + * @default false + */ + allow_non_billable_realtime_protocols: boolean; /** * Allow Unmanaged Response Ids * @description If True, lets keys address Responses API ids that this proxy did not issue (raw provider ids, or ids issued before response-id encryption was configured). Such an id carries no owner, so no ownership check can run on it; ids this proxy did issue keep full ownership enforcement. Off by default, in which case an unrecognized response id is rejected with 403 @@ -56324,7 +56492,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_3: { + websocket_realtime_websocket_endpoint_get_6: { parameters: { query?: never; header?: never; @@ -56402,6 +56570,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get_3: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_openai_v1_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; responses_api_openai_v1_responses_post: { parameters: { query?: never; @@ -59409,7 +59635,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get: { + websocket_realtime_websocket_endpoint_get_4: { parameters: { query?: never; header?: never; @@ -59487,6 +59713,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; register_client_register_post: { parameters: { query?: { @@ -68594,7 +68878,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_2: { + websocket_realtime_websocket_endpoint_get_5: { parameters: { query?: never; header?: never; @@ -68672,6 +68956,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_v1_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_v1_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; rerank_v1_rerank_post: { parameters: { query?: never; From db3e6f06a0c2d40a18bffffa02d2992030919936 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 18:35:46 -0500 Subject: [PATCH 08/21] fix(realtime): drop unused tiktoken import --- litellm/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 3d3a0bb7ec6..e6f451fa4c6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,7 +37,6 @@ if TYPE_CHECKING: import dotenv import httpx import openai -import tiktoken from openai import AsyncStream, Stream from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel From 39ac30122ecd0e4612e5ffe28509173ee2c2ebee Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 18:59:55 -0500 Subject: [PATCH 09/21] fix(realtime): preserve provider duration and model capabilities --- ci_cd/generate_model_prices_schema.py | 5 +++ cookbook/gpt_realtime_translate.py | 6 ++- .../litellm_core_utils/realtime_streaming.py | 17 ++++++++ litellm/main.py | 36 +++++++++++++--- ...odel_prices_and_context_window_backup.json | 6 +++ litellm/types/utils.py | 1 + litellm/utils.py | 3 ++ model_prices_and_context_window.json | 6 +++ model_prices_and_context_window.schema.json | 7 +++ .../test_realtime_streaming.py | 29 ++++++++++++- .../cookbook/test_gpt_realtime_translate.py | 43 +++++++++++++++++++ 11 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 tests/unit/cookbook/test_gpt_realtime_translate.py diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 8eec07dadda..06971107200 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -116,6 +116,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = { "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", "items": STRING, }, + "supported_transcription_response_formats": { + "type": "array", + "description": "Response formats accepted by the model for file transcription.", + "items": STRING, + }, "supported_modalities": { "type": "array", "description": "Input modalities the model accepts.", diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py index 73c91eaa1e9..bae89b829ca 100644 --- a/cookbook/gpt_realtime_translate.py +++ b/cookbook/gpt_realtime_translate.py @@ -176,7 +176,11 @@ async def receive_translation( output.setframerate(SAMPLE_RATE) write_stdout("Translation: ", end="", flush=True) while True: - timeout = OUTPUT_IDLE_TIMEOUT_SECONDS if sender_finished.is_set() else INITIAL_RESPONSE_TIMEOUT_SECONDS + timeout = ( + OUTPUT_IDLE_TIMEOUT_SECONDS + if sender_finished.is_set() and audio_received.is_set() + else INITIAL_RESPONSE_TIMEOUT_SECONDS + ) try: raw_event = await asyncio.wait_for(connection.recv(), timeout=timeout) except TimeoutError: diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 47ef8369e56..81b711df529 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -437,6 +437,21 @@ class RealTimeStreaming: def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None: if not self._is_translation_session: return + if event_obj.get("type") == "session.closed": + usage: Final = event_obj.get("usage") + output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None + if isinstance(output_seconds, (int, float)): + if not self._should_store_message(event_obj): + self.messages.append( + OpenAIRealtimeTranslationClosedEvent( + type="session.closed", + usage=OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=output_seconds + ), + ) + ) + self._translation_usage_finalized = True + return self._capture_translation_output_format(event_obj) if event_obj.get("type") not in ( "session.output_audio.delta", @@ -1117,6 +1132,8 @@ class RealTimeStreaming: for event in events: if self._should_drop_event_from_client(event): continue + if isinstance(event, dict): + self._capture_translation_output_audio(event) is_session_created_event = isinstance(event, dict) and event.get("type") == "session.created" if is_session_created_event: if self._uses_deferred_backend_setup() and not self._backend_setup_complete: diff --git a/litellm/main.py b/litellm/main.py index e6f451fa4c6..a9ba9fc2455 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -23,7 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy -from functools import partial +from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args from urllib.parse import urlsplit @@ -39,7 +39,7 @@ import httpx import openai from openai import AsyncStream, Stream from openai.types.audio import TranscriptionStreamEvent -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter from typing_extensions import overload import litellm @@ -7871,6 +7871,23 @@ async def atranscription( ) +@lru_cache(maxsize=1) +def _bundled_transcription_response_formats() -> Mapping[str, tuple[str, ...]]: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + catalog: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python( + GetModelCostMap.load_local_model_cost_map() + ) + formats_adapter: Final = TypeAdapter(tuple[str, ...]) + return MappingProxyType( + { + model: formats_adapter.validate_python(entry["supported_transcription_response_formats"]) + for model, entry in catalog.items() + if "supported_transcription_response_formats" in entry + } + ) + + def _validate_gpt_transcription_request( model: str, custom_llm_provider: str, @@ -7879,25 +7896,30 @@ def _validate_gpt_transcription_request( response_format: str | None, api_version: str | None, ) -> str | None: + model_info: Final = get_model_info(model=model) if model in litellm.model_cost else {} + supported_endpoints: Final = model_info.get("supported_endpoints") + supported_formats: Final = model_info.get("supported_transcription_response_formats") or ( + _bundled_transcription_response_formats().get(model) + ) if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", model=model, llm_provider=custom_llm_provider, ) - if model == "gpt-live-transcribe": + if supported_endpoints is not None and "/v1/audio/transcriptions" not in supported_endpoints: raise litellm.UnsupportedParamsError( - message="gpt-live-transcribe is available through the Realtime API, not file transcription", + message=f"{model} is available through the Realtime API, not file transcription", model=model, llm_provider=custom_llm_provider, ) - if model == "gpt-transcribe" and response_format not in (None, "json"): + if supported_formats is not None and response_format is not None and response_format not in supported_formats: raise litellm.UnsupportedParamsError( - message="gpt-transcribe only supports response_format='json'", + message=f"{model} only supports response_format={', '.join(repr(fmt) for fmt in supported_formats)}", model=model, llm_provider=custom_llm_provider, ) - if custom_llm_provider == "azure" and model == "gpt-transcribe": + if custom_llm_provider == "azure" and supported_formats is not None: if api_version in (None, "v1", "latest", "preview"): return litellm.AZURE_DEFAULT_API_VERSION return api_version diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 83525938f23..5fa391fc23d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -214,6 +214,9 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "audio", "text" @@ -57555,6 +57558,9 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "text", "audio" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6667e2ee400..0846209e749 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -396,6 +396,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): ] ] supported_endpoints: list[str] | None + supported_transcription_response_formats: ReadOnly[Sequence[str] | None] use_openai_responses_path: bool | None tpm: int | None rpm: int | None diff --git a/litellm/utils.py b/litellm/utils.py index ba00ae4c6d3..9b4a64eb3c4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6235,6 +6235,9 @@ def _get_model_info_helper( litellm_provider=_model_info.get("litellm_provider", custom_llm_provider), mode=_model_info.get("mode"), supported_endpoints=_model_info.get("supported_endpoints", None), + supported_transcription_response_formats=_model_info.get( + "supported_transcription_response_formats", None + ), supports_system_messages=_model_info.get("supports_system_messages", None), supports_response_schema=_model_info.get("supports_response_schema", None), supports_vision=_model_info.get("supports_vision", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 83525938f23..5fa391fc23d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -214,6 +214,9 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "audio", "text" @@ -57555,6 +57558,9 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], + "supported_transcription_response_formats": [ + "json" + ], "supported_modalities": [ "text", "audio" diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 737a9b7fa60..cac91e189ab 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -864,6 +864,13 @@ "type": "string" } }, + "supported_transcription_response_formats": { + "type": "array", + "description": "Response formats accepted by the model for file transcription.", + "items": { + "type": "string" + } + }, "supports_adaptive_thinking": { "type": "boolean" }, diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 33493ffe6c1..9fcf5b1b630 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -1073,7 +1073,7 @@ async def test_translation_session_update_rejects_disallowed_nested_transcriptio translation_session=True, ) - with pytest.raises(Exception, match="Tried to access gpt-live-transcribe"): + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available"): await streaming._send_to_backend( json.dumps( { @@ -2910,7 +2910,9 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] -@pytest.mark.parametrize("event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"]) +@pytest.mark.parametrize( + "event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"] +) def test_translation_audio_duration_is_finalized_once(event_type: str): import base64 @@ -2974,6 +2976,29 @@ def test_translation_does_not_duplicate_provider_duration_usage(): assert len(closed_events) == 1 +def test_translation_prefers_provider_duration_over_audio_byte_estimate(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 0.5} + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/unit/cookbook/test_gpt_realtime_translate.py b/tests/unit/cookbook/test_gpt_realtime_translate.py new file mode 100644 index 00000000000..e87005d1bad --- /dev/null +++ b/tests/unit/cookbook/test_gpt_realtime_translate.py @@ -0,0 +1,43 @@ +import asyncio +import base64 +import json +import wave +from pathlib import Path +from types import SimpleNamespace +from typing import Final, cast + +import pytest +from websockets.asyncio.client import ClientConnection + +from cookbook import gpt_realtime_translate as translate + + +@pytest.mark.asyncio +async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(translate, "OUTPUT_IDLE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(translate, "INITIAL_RESPONSE_TIMEOUT_SECONDS", 0.1) + audio: Final = bytes(480) + events: Final = iter( + ( + {"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}, + {"type": "error", "error": {"message": "session closed"}}, + ) + ) + + async def recv() -> str: + event: Final = next(events) + if event["type"] == "session.output_audio.delta": + await asyncio.sleep(0.03) + return json.dumps(event) + + sender_finished: Final = asyncio.Event() + sender_finished.set() + output: Final = tmp_path / "translation.wav" + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished + ) + + assert result == 'Realtime API error: {"message": "session closed"}' + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(240) == audio From 34d07730763bb59aa1159b752d9b3a90c9c55a0a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 19:16:22 -0500 Subject: [PATCH 10/21] fix(realtime): keep transcription validation compatible with cost map guard --- ci_cd/generate_model_prices_schema.py | 5 ---- litellm/main.py | 30 ++++--------------- ...odel_prices_and_context_window_backup.json | 6 ---- litellm/types/utils.py | 1 - litellm/utils.py | 3 -- model_prices_and_context_window.json | 6 ---- model_prices_and_context_window.schema.json | 7 ----- 7 files changed, 5 insertions(+), 53 deletions(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 06971107200..8eec07dadda 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -116,11 +116,6 @@ ARRAY_KEYS: dict[str, JsonSchema] = { "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", "items": STRING, }, - "supported_transcription_response_formats": { - "type": "array", - "description": "Response formats accepted by the model for file transcription.", - "items": STRING, - }, "supported_modalities": { "type": "array", "description": "Input modalities the model accepts.", diff --git a/litellm/main.py b/litellm/main.py index a9ba9fc2455..ad2ea8c5c77 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -23,7 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy -from functools import lru_cache, partial +from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args from urllib.parse import urlsplit @@ -39,7 +39,7 @@ import httpx import openai from openai import AsyncStream, Stream from openai.types.audio import TranscriptionStreamEvent -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel from typing_extensions import overload import litellm @@ -7871,23 +7871,6 @@ async def atranscription( ) -@lru_cache(maxsize=1) -def _bundled_transcription_response_formats() -> Mapping[str, tuple[str, ...]]: - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - - catalog: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python( - GetModelCostMap.load_local_model_cost_map() - ) - formats_adapter: Final = TypeAdapter(tuple[str, ...]) - return MappingProxyType( - { - model: formats_adapter.validate_python(entry["supported_transcription_response_formats"]) - for model, entry in catalog.items() - if "supported_transcription_response_formats" in entry - } - ) - - def _validate_gpt_transcription_request( model: str, custom_llm_provider: str, @@ -7898,9 +7881,6 @@ def _validate_gpt_transcription_request( ) -> str | None: model_info: Final = get_model_info(model=model) if model in litellm.model_cost else {} supported_endpoints: Final = model_info.get("supported_endpoints") - supported_formats: Final = model_info.get("supported_transcription_response_formats") or ( - _bundled_transcription_response_formats().get(model) - ) if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", @@ -7913,13 +7893,13 @@ def _validate_gpt_transcription_request( model=model, llm_provider=custom_llm_provider, ) - if supported_formats is not None and response_format is not None and response_format not in supported_formats: + if model == "gpt-transcribe" and response_format not in (None, "json"): raise litellm.UnsupportedParamsError( - message=f"{model} only supports response_format={', '.join(repr(fmt) for fmt in supported_formats)}", + message="gpt-transcribe only supports response_format='json'", model=model, llm_provider=custom_llm_provider, ) - if custom_llm_provider == "azure" and supported_formats is not None: + if custom_llm_provider == "azure" and model == "gpt-transcribe": if api_version in (None, "v1", "latest", "preview"): return litellm.AZURE_DEFAULT_API_VERSION return api_version diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5fa391fc23d..83525938f23 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -214,9 +214,6 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "audio", "text" @@ -57558,9 +57555,6 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "text", "audio" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0846209e749..6667e2ee400 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -396,7 +396,6 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): ] ] supported_endpoints: list[str] | None - supported_transcription_response_formats: ReadOnly[Sequence[str] | None] use_openai_responses_path: bool | None tpm: int | None rpm: int | None diff --git a/litellm/utils.py b/litellm/utils.py index 9b4a64eb3c4..ba00ae4c6d3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6235,9 +6235,6 @@ def _get_model_info_helper( litellm_provider=_model_info.get("litellm_provider", custom_llm_provider), mode=_model_info.get("mode"), supported_endpoints=_model_info.get("supported_endpoints", None), - supported_transcription_response_formats=_model_info.get( - "supported_transcription_response_formats", None - ), supports_system_messages=_model_info.get("supports_system_messages", None), supports_response_schema=_model_info.get("supports_response_schema", None), supports_vision=_model_info.get("supports_vision", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5fa391fc23d..83525938f23 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -214,9 +214,6 @@ "/v1/realtime", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "audio", "text" @@ -57558,9 +57555,6 @@ "/v1/audio/transcriptions", "/v1/realtime/transcription_sessions" ], - "supported_transcription_response_formats": [ - "json" - ], "supported_modalities": [ "text", "audio" diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index cac91e189ab..737a9b7fa60 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -864,13 +864,6 @@ "type": "string" } }, - "supported_transcription_response_formats": { - "type": "array", - "description": "Response formats accepted by the model for file transcription.", - "items": { - "type": "string" - } - }, "supports_adaptive_thinking": { "type": "boolean" }, From 17dbd6ae927bed575ee7c407671a2f4c7d9c3dad Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 20:16:11 -0500 Subject: [PATCH 11/21] fix(realtime): address transcription and billing review findings --- litellm/__init__.py | 2 - litellm/constants.py | 31 ----- litellm/cost_calculator.py | 34 ++++-- .../audio_utils/transcription_streaming.py | 4 + .../litellm_core_utils/realtime_streaming.py | 5 +- litellm/llms/azure/audio_transcriptions.py | 6 +- litellm/llms/azure/realtime/handler.py | 19 +++- .../azure/realtime/http_transformation.py | 21 +++- .../audio_transcription/transformation.py | 17 +++ .../base_llm/realtime/http_transformation.py | 11 ++ litellm/llms/openai/transcriptions/handler.py | 22 +--- litellm/main.py | 16 ++- ...odel_prices_and_context_window_backup.json | 41 +++++++ litellm/realtime_api/main.py | 52 +++------ litellm/types/llms/openai.py | 1 + model_prices_and_context_window.json | 41 +++++++ .../test_realtime_streaming.py | 4 +- .../realtime/test_transcription_sessions.py | 10 +- .../llms/openai/realtime/test_translation.py | 12 ++ .../public_endpoints/test_public_endpoints.py | 35 ++---- tests/test_litellm/test_cost_calculator.py | 20 ++++ .../transcriptions/test_gpt_transcribe.py | 106 ++++++++++++++++-- tests/unit/realtime_api/test_main.py | 23 ++++ 23 files changed, 384 insertions(+), 149 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 4ec7aeea8fb..cfe62c28561 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -72,7 +72,6 @@ from litellm.constants import ( OPENAI_CHAT_COMPLETION_PARAMS as _openai_completion_params, # backwards compatibility OPENAI_FINISH_REASONS, OPENAI_FINISH_REASONS as _openai_finish_reasons, # backwards compatibility - OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS, openai_compatible_endpoints, openai_compatible_providers, openai_text_completion_compatible_providers, @@ -1008,7 +1007,6 @@ def add_known_models(model_cost_map: Optional[Dict] = None): _populate_provider_model_sets(model_cost) -open_ai_chat_completion_models.update(OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS) # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary # this is maintained for Exception Mapping diff --git a/litellm/constants.py b/litellm/constants.py index 855f51811a4..b6b08f25b93 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -835,37 +835,6 @@ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "timestamp_granularities", ] -OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS: Final = frozenset( - { - "gpt-realtime-2", - "gpt-realtime-2.1", - "gpt-realtime-2.1-mini", - "gpt-realtime-translate", - "gpt-realtime-whisper", - "gpt-transcribe", - "gpt-live-transcribe", - } -) - -AZURE_GA_REALTIME_MODELS: Final = frozenset( - { - "gpt-realtime-2", - "gpt-realtime-2-2026-05-06", - "gpt-realtime-2.1", - "gpt-realtime-2.1-2026-07-07", - "gpt-realtime-2.1-mini", - "gpt-realtime-2.1-mini-2026-07-07", - "gpt-realtime-translate", - "gpt-realtime-translate-2026-05-06", - "gpt-realtime-translate-2026-05-07", - "gpt-realtime-whisper", - "gpt-realtime-whisper-2026-05-06", - "gpt-realtime-whisper-2026-05-07", - "gpt-transcribe", - "gpt-live-transcribe", - } -) - OPENAI_EMBEDDING_PARAMS: Final = ["dimensions", "encoding_format", "user"] DEFAULT_EMBEDDING_PARAM_VALUES: Final = { diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a256bc85425..98af08f6472 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -3023,14 +3023,21 @@ def handle_realtime_translation_cost_calculation( custom_llm_provider: str, litellm_model_name: str, ) -> float: - output_seconds = 0.0 # rebind-ok: duration is accumulated across translation close events - for result in results: - if result.get("type") != _TRANSLATION_CLOSED_EVENT_TYPE: - continue - usage = result.get("usage") - if isinstance(usage, dict) and isinstance(usage.get("output_seconds"), (int, float)): - output_seconds += float(usage["output_seconds"]) - if output_seconds <= 0: + usage_events: Final = tuple( + usage + for result in results + if result.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE + if isinstance(usage := result.get("usage"), dict) + ) + input_seconds: Final = sum( + float(usage["input_seconds"]) for usage in usage_events if isinstance(usage.get("input_seconds"), (int, float)) + ) + output_seconds: Final = sum( + float(usage["output_seconds"]) + for usage in usage_events + if isinstance(usage.get("output_seconds"), (int, float)) + ) + if input_seconds <= 0 and output_seconds <= 0: return 0.0 try: model_info: Final = litellm.get_model_info( @@ -3039,10 +3046,15 @@ def handle_realtime_translation_cost_calculation( ) except Exception: # noqa: BLE001 # unknown model metadata should yield zero translation cost return 0.0 + input_cost_per_second: Final = model_info.get("input_cost_per_second") output_cost_per_second: Final = model_info.get("output_cost_per_second") - if not isinstance(output_cost_per_second, (int, float)): - return 0.0 - return output_seconds * output_cost_per_second + input_cost: Final = ( + input_seconds * input_cost_per_second if isinstance(input_cost_per_second, (int, float)) else 0.0 + ) + output_cost: Final = ( + output_seconds * output_cost_per_second if isinstance(output_cost_per_second, (int, float)) else 0.0 + ) + return input_cost + output_cost def handle_realtime_transcription_cost_calculation( diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 5ae246f3468..979a1847362 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -120,6 +120,8 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): if self._finalized or self._failed: return self._finalized = True + if self._collector.done_event is None and self._collector.duration is None: + return self._logging_obj.success_handler( self._collector.response(), self._start_time, @@ -174,6 +176,8 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): if self._finalized or self._failed: return self._finalized = True + if self._collector.done_event is None and self._collector.duration is None: + return response: Final = self._collector.response() end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract self._logging_obj.handle_sync_success_callbacks_for_async_calls( diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 81b711df529..cc206079c48 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -441,12 +441,15 @@ class RealTimeStreaming: usage: Final = event_obj.get("usage") output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None if isinstance(output_seconds, (int, float)): + input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None if not self._should_store_message(event_obj): self.messages.append( OpenAIRealtimeTranslationClosedEvent( type="session.closed", usage=OpenAIRealtimeTranslationDurationUsage( - type="duration", output_seconds=output_seconds + type="duration", + output_seconds=output_seconds, + **({"input_seconds": input_seconds} if isinstance(input_seconds, (int, float)) else {}), ), ) ) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index fb575a3713c..a6f2b900615 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -6,6 +6,7 @@ from pydantic import BaseModel from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name +from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data from litellm.types.utils import FileTypes from litellm.utils import ( TranscriptionResponse, @@ -40,11 +41,12 @@ class AzureAudioTranscription(AzureChatCompletion): custom_llm_provider: str = "azure", ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} + sdk_data: Final = sdk_compatible_transcription_request_data(data) if atranscription is True: return self.async_audio_transcriptions( audio_file=audio_file, - data=data, + data=sdk_data, model_response=model_response, timeout=timeout, api_key=api_key, @@ -86,7 +88,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) response: Final = azure_client.audio.transcriptions.create( - **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 393499c0fcd..d3b7ba6910f 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -8,6 +8,7 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, Protocol, cast +import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams @@ -39,16 +40,30 @@ async def forward_messages(client_ws: Any, backend_ws: Any): pass +def azure_realtime_requires_ga(model: str) -> bool: + try: + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + return False + return (model_info.get("provider_specific_entry") or {}).get("realtime_ga_only") == 1 + + def azure_realtime_protocol_for_client( configured_protocol: object, *, + model: str, + realtime_mode: str, query_params: RealtimeQueryParams | None, websocket: ScopedWebSocket, ) -> str: + if azure_realtime_requires_ga(model): + if isinstance(configured_protocol, str) and configured_protocol.upper() not in ("GA", "V1"): + raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") + return "GA" + if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": + return "GA" if isinstance(configured_protocol, str) and configured_protocol: return configured_protocol - if (query_params or {}).get("intent") == "transcription": - return "GA" return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index aa2b6d9cda0..9b37d1faa95 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -1,17 +1,21 @@ """Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" +from collections.abc import Mapping from typing import Final import litellm -from litellm.constants import AZURE_GA_REALTIME_MODELS +from litellm.llms.azure.common_utils import get_azure_ad_token from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +from .handler import azure_realtime_requires_ga class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): @staticmethod def _uses_ga_api(model: str, api_version: str | None) -> bool: - return api_version in ("preview", "latest", "v1") or model in AZURE_GA_REALTIME_MODELS + return api_version in ("preview", "latest", "v1") or azure_realtime_requires_ga(model) def get_api_base(self, api_base: str | None, **kwargs) -> str: return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or "" @@ -19,6 +23,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): def get_api_key(self, api_key: str | None, **kwargs) -> str: return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or "" + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + if api_key: + return extra_headers + azure_ad_token: Final = get_azure_ad_token(litellm_params) + if azure_ad_token is None: + return extra_headers + return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"} + def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") if self._uses_ga_api(model, api_version): diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..2078b063453 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -11,6 +11,23 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse + +def sdk_compatible_transcription_request_data(data: dict) -> dict: + extension_keys: Final = ("keywords", "languages") + extension_body: Final = {key: data[key] for key in extension_keys if key in data} + if not extension_body: + return data + + existing_extra_body: Final = data.get("extra_body") + return { + **{key: value for key, value in data.items() if key not in extension_keys}, + "extra_body": { + **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), + **extension_body, + }, + } + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index 0d64ed3931d..de5032be381 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow: """ from abc import ABC, abstractmethod +from collections.abc import Mapping from typing import Final import httpx +from litellm.types.router import GenericLiteLLMParams + class BaseRealtimeHTTPConfig(ABC): """ @@ -49,6 +52,14 @@ class BaseRealtimeHTTPConfig(ABC): explicit api_key → litellm.api_key → env var → "" """ + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + return extra_headers + # ------------------------------------------------------------------ # # client_secrets endpoint # # ------------------------------------------------------------------ # diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 2c703a16d4e..1e30ffbb43d 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -12,6 +12,7 @@ from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, + sdk_compatible_transcription_request_data, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -25,23 +26,6 @@ from ..openai import OpenAIChatCompletion class OpenAIAudioTranscription(OpenAIChatCompletion): # Audio Transcriptions - @staticmethod - def _sdk_compatible_request_data(data: dict) -> dict: - """Route API fields that predate SDK support through ``extra_body``.""" - extension_keys: Final = ("keywords", "languages") - extension_body: Final = {key: data[key] for key in extension_keys if key in data} - if not extension_body: - return data - - existing_extra_body: Final = data.get("extra_body") - return { # mutable-ok: OpenAI SDK requires a mutable request mapping - **{key: value for key, value in data.items() if key not in extension_keys}, - "extra_body": { - **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), - **extension_body, - }, - } - async def make_openai_audio_transcriptions_request( self, openai_aclient: AsyncOpenAI, @@ -54,7 +38,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): - call openai_aclient.audio.transcriptions.create by default """ try: - sdk_data: Final = self._sdk_compatible_request_data(data) + sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout) return {}, stream_response # mutable-ok: response headers use the existing mutable mapping contract @@ -80,7 +64,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): - call openai_aclient.audio.transcriptions.create by default """ try: - sdk_data: Final = self._sdk_compatible_request_data(data) + sdk_data: Final = sdk_compatible_transcription_request_data(data) if data.get("stream") is True: response = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) return None, response diff --git a/litellm/main.py b/litellm/main.py index ad2ea8c5c77..0f644bcff5a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7879,8 +7879,14 @@ def _validate_gpt_transcription_request( response_format: str | None, api_version: str | None, ) -> str | None: - model_info: Final = get_model_info(model=model) if model in litellm.model_cost else {} + model_cost_key: Final = f"{custom_llm_provider}/{model}" if custom_llm_provider == "azure" else model + model_info: Final = ( + get_model_info(model=model, custom_llm_provider=custom_llm_provider) + if model_cost_key in litellm.model_cost + else {} + ) supported_endpoints: Final = model_info.get("supported_endpoints") + provider_specific_entry: Final = model_info.get("provider_specific_entry") or {} if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", @@ -7893,14 +7899,14 @@ def _validate_gpt_transcription_request( model=model, llm_provider=custom_llm_provider, ) - if model == "gpt-transcribe" and response_format not in (None, "json"): + if provider_specific_entry.get("transcription_json_only") == 1 and response_format not in (None, "json"): raise litellm.UnsupportedParamsError( - message="gpt-transcribe only supports response_format='json'", + message=f"{model} only supports response_format='json'", model=model, llm_provider=custom_llm_provider, ) - if custom_llm_provider == "azure" and model == "gpt-transcribe": - if api_version in (None, "v1", "latest", "preview"): + if provider_specific_entry.get("transcription_deployment_api") == 1: + if api_version in ("v1", "latest", "preview"): return litellm.AZURE_DEFAULT_API_VERSION return api_version return api_version diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 83525938f23..586cfdc59e5 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3,6 +3,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/realtime", @@ -30,6 +33,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -67,6 +73,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -98,6 +107,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -122,6 +134,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -146,6 +161,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", "supported_endpoints": [ @@ -168,6 +186,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -188,6 +209,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -208,6 +232,11 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -6057,6 +6086,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6095,6 +6127,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6290,6 +6325,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -57550,6 +57588,9 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_only": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 02055988565..bc461d0087e 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -10,7 +10,6 @@ import httpx import litellm from litellm.constants import ( - AZURE_GA_REALTIME_MODELS, AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, @@ -51,8 +50,6 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket - from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig - azure_realtime: Final = AzureOpenAIRealtime() openai_realtime: Final = OpenAIRealtime() bedrock_realtime: Final = BedrockRealtime() @@ -112,21 +109,6 @@ def _build_litellm_metadata(kwargs: dict) -> dict: return metadata -def _resolve_azure_realtime_protocol( - model: str, - realtime_protocol: str | None, - query_params: RealtimeQueryParams | None, - realtime_mode: str, -) -> str: - if model in AZURE_GA_REALTIME_MODELS: - if realtime_protocol is not None and realtime_protocol.upper() not in ("GA", "V1"): - raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") - return "GA" - if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": - return "GA" - return realtime_protocol or "beta" - - def _get_realtime_http_provider_config( custom_llm_provider: str, dynamic_api_base: str | None, @@ -164,19 +146,18 @@ def _get_realtime_http_provider_config( def _get_realtime_http_extra_headers( - custom_llm_provider: str, + provider_config: BaseRealtimeHTTPConfig | None, litellm_params: GenericLiteLLMParams, resolved_api_key: str, extra_headers: Mapping[str, object] | None, ) -> Mapping[str, object] | None: - resolved_headers: Final = { # mutable-ok: Azure authentication may extend caller-supplied headers - **(extra_headers or {}) - } - if custom_llm_provider == "azure" and not resolved_api_key: - azure_ad_token: Final = get_azure_ad_token(litellm_params) - if azure_ad_token: - resolved_headers["Authorization"] = f"Bearer {azure_ad_token}" - return resolved_headers or None + if provider_config is None: + return extra_headers + return provider_config.get_extra_headers( + litellm_params=litellm_params, + api_key=resolved_api_key, + extra_headers=extra_headers, + ) @wrapper_client @@ -229,7 +210,7 @@ async def acreate_realtime_client_secret( litellm_params=litellm_params, ) resolved_extra_headers: Final = _get_realtime_http_extra_headers( - custom_llm_provider=custom_llm_provider, + provider_config=provider_config, litellm_params=litellm_params, resolved_api_key=resolved_api_key, extra_headers=kwargs.get("extra_headers"), @@ -302,7 +283,7 @@ async def acreate_realtime_translation_client_secret( litellm_params=litellm_params, ) resolved_extra_headers: Final = _get_realtime_http_extra_headers( - custom_llm_provider=custom_llm_provider, + provider_config=provider_config, litellm_params=litellm_params, resolved_api_key=resolved_api_key, extra_headers=kwargs.get("extra_headers"), @@ -385,7 +366,7 @@ async def acreate_realtime_transcription_session( litellm_params=litellm_params, ) resolved_extra_headers: Final = _get_realtime_http_extra_headers( - custom_llm_provider=custom_llm_provider, + provider_config=provider_config, litellm_params=litellm_params, resolved_api_key=resolved_api_key, extra_headers=kwargs.get("extra_headers"), @@ -660,17 +641,12 @@ async def _arealtime( or litellm_params.get("realtime_protocol") or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") ) - realtime_protocol: Final = _resolve_azure_realtime_protocol( + realtime_protocol: Final = azure_realtime_protocol_for_client( + configured_realtime_protocol, model=model, - realtime_protocol=( - configured_realtime_protocol - if model in AZURE_GA_REALTIME_MODELS or realtime_mode == "translation" - else azure_realtime_protocol_for_client( - configured_realtime_protocol, query_params=query_params, websocket=websocket - ) - ), query_params=query_params, realtime_mode=realtime_mode, + websocket=websocket, ) await azure_realtime.async_realtime( model=model, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 083b7beda41..9c3cf6b25a1 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2314,6 +2314,7 @@ class OpenAIRealtimeResponseUsage(TypedDict): class OpenAIRealtimeTranslationDurationUsage(TypedDict): type: ReadOnly[Literal["duration"]] output_seconds: ReadOnly[float] + input_seconds: NotRequired[ReadOnly[float]] class OpenAIRealtimeTranslationClosedEvent(TypedDict): diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 83525938f23..586cfdc59e5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3,6 +3,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/realtime", @@ -30,6 +33,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -67,6 +73,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -98,6 +107,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -122,6 +134,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", "supported_endpoints": [ @@ -146,6 +161,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_second": 0.0005666666666666667, "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", "supported_endpoints": [ @@ -168,6 +186,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -188,6 +209,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -208,6 +232,11 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -6057,6 +6086,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6095,6 +6127,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6290,6 +6325,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -57550,6 +57588,9 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_only": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 9fcf5b1b630..f7473787cd1 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -2990,13 +2990,13 @@ def test_translation_prefers_provider_duration_over_audio_byte_estimate(): {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} ) streaming._capture_translation_output_audio( - {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}} + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5}} ) streaming._finalize_translation_usage() closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] assert len(closed_events) == 1 - assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 0.5} + assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5} @pytest.mark.asyncio diff --git a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py index c663efa341b..0c3630bb107 100644 --- a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py @@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import litellm from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -46,15 +47,18 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version(): @pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) -def test_azure_ga_realtime_http_urls(api_version): +def test_azure_ga_realtime_http_urls(api_version, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() cfg = AzureRealtimeHTTPConfig() base = "https://my.openai.azure.com" - assert cfg.get_complete_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/client_secrets") - assert cfg.get_realtime_calls_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/calls") + assert cfg.get_complete_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/calls") assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == ( f"{base}/openai/v1/realtime/transcription_sessions" ) + litellm.get_model_info.cache_clear() def test_request_resolves_model_returns_none_when_both_absent(): diff --git a/tests/test_litellm/llms/openai/realtime/test_translation.py b/tests/test_litellm/llms/openai/realtime/test_translation.py index a329200b2f7..06af49f26ca 100644 --- a/tests/test_litellm/llms/openai/realtime/test_translation.py +++ b/tests/test_litellm/llms/openai/realtime/test_translation.py @@ -11,6 +11,18 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig from litellm.types.realtime import RealtimeSessionConfig +from litellm.types.router import GenericLiteLLMParams + + +def test_azure_realtime_http_headers_use_ad_token_without_api_key(): + config = AzureRealtimeHTTPConfig() + params = GenericLiteLLMParams(azure_ad_token="test-ad-token") + + assert config.get_extra_headers(params, "", {"X-Trace": "trace"}) == { + "X-Trace": "trace", + "Authorization": "Bearer test-ad-token", + } + assert config.get_extra_headers(params, "test-api-key", {"X-Trace": "trace"}) == {"X-Trace": "trace"} def test_realtime_session_config_supports_translation_and_live_transcription_fields(): diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0dec44af402..99f4f909815 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -5,11 +5,10 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest - - from fastapi import FastAPI from fastapi.testclient import TestClient +import litellm from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.public_endpoints import router from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets @@ -74,32 +73,20 @@ def test_get_provider_create_fields(): ), "Expected at least one provider to have detailed credential fields" -def test_get_litellm_model_cost_map_returns_cost_map(): - app = FastAPI() +def test_get_litellm_model_cost_map_returns_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + model_cost: Final = { + "audio-model": {"input_cost_per_second": 0.1, "mode": "audio_transcription"}, + "token-model": {"input_cost_per_token": 0.01, "mode": "chat"}, + } + monkeypatch.setattr(litellm, "model_cost", model_cost) + app: Final = FastAPI() app.include_router(router) - client = TestClient(app) + client: Final = TestClient(app) - response = client.get("/public/litellm_model_cost_map") + response: Final = client.get("/public/litellm_model_cost_map") assert response.status_code == 200 - payload = response.json() - assert isinstance(payload, dict) - assert len(payload) > 0, "Expected model cost map to contain at least one model" - - # Verify the structure contains expected keys for at least one model - # Check for a common model like gpt-4 or gpt-3.5-turbo - model_keys = list(payload.keys()) - assert len(model_keys) > 0 - - # Verify at least one model has expected cost fields - sample_model = model_keys[0] - sample_model_data = payload[sample_model] - assert isinstance(sample_model_data, dict) - # Check for common cost fields that should be present - assert ( - "input_cost_per_token" in sample_model_data - or "output_cost_per_token" in sample_model_data - ) + assert response.json() == model_cost def test_public_ai_hub_info_is_public_by_default(monkeypatch): diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7362d36d523..bcb0fd15760 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4685,3 +4685,23 @@ def test_realtime_translation_duration_cost(_local_model_cost_map): ) assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) + + +def test_realtime_translation_duration_cost_includes_provider_input_usage(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 3.0, "output_seconds": 2.0}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + expected: Final = ( + 3 * litellm.model_cost[model]["input_cost_per_second"] + + 2 * litellm.model_cost[model]["output_cost_per_second"] + ) + assert actual == pytest.approx(expected) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 35bed209400..742e1852ea5 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -1,12 +1,13 @@ import io import json import wave +from collections.abc import Iterator from datetime import datetime from unittest.mock import AsyncMock, MagicMock import httpx import pytest -from openai import AsyncOpenAI, AsyncStream, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI import litellm from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream @@ -20,6 +21,14 @@ from litellm.types.utils import TranscriptionResponse from litellm.utils import get_optional_params_transcription +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + def test_gpt_transcribe_config_uses_native_parameters_and_json(): config = OpenAIGPTTranscribeAudioTranscriptionConfig() supported = config.get_supported_openai_params("gpt-transcribe") @@ -122,6 +131,37 @@ async def test_openai_handler_returns_native_typed_stream(): assert logged_response.languages == [] +@pytest.mark.asyncio +async def test_closed_transcription_stream_without_usage_or_duration_does_not_log_success(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello"}, + {"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 1}}, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + stream = await client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now()) + + async for event in wrapped_stream: + assert event.type == "transcript.text.delta" + break + await wrapped_stream.close() + await client.close() + + logging_obj.handle_sync_success_callbacks_for_async_calls.assert_not_called() + logging_obj.async_success_handler.assert_not_awaited() + logging_obj.async_failure_handler.assert_not_awaited() + + @pytest.mark.asyncio async def test_atranscription_stream_preserves_duration_for_callback_cost(): async def send_response(request: httpx.Request) -> httpx.Response: @@ -188,19 +228,20 @@ def test_gpt_transcribe_rejects_conflicting_language_inputs(): ) -def test_gpt_transcribe_rejects_whisper_response_formats(): +@pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"]) +def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None: audio_file = io.BytesIO(b"audio") audio_file.name = "sample.wav" with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format='json'"): litellm.transcription( - model="gpt-transcribe", + model=model, file=audio_file, response_format="verbose_json", api_key="sk-test", ) -def test_gpt_live_transcribe_rejects_file_transcription(): +def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: None) -> None: audio_file = io.BytesIO(b"audio") audio_file.name = "sample.wav" with pytest.raises(litellm.UnsupportedParamsError, match="Realtime API"): @@ -232,8 +273,8 @@ def test_azure_async_gpt_transcribe_forwards_v1_api_version(): assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == "v1" -@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) -def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: str | None): +@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) +def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(local_model_cost_map: None, api_version: str) -> None: resolved_api_version = _validate_gpt_transcription_request( model="gpt-transcribe", custom_llm_provider="azure", @@ -246,6 +287,20 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: st assert resolved_api_version == litellm.AZURE_DEFAULT_API_VERSION +def test_azure_gpt_transcribe_keeps_unset_api_version_for_configured_default(local_model_cost_map: None) -> None: + assert ( + _validate_gpt_transcription_request( + model="gpt-transcribe", + custom_llm_provider="azure", + language=None, + languages=None, + response_format="json", + api_version=None, + ) + is None + ) + + def test_azure_gpt_transcribe_uses_deployment_scoped_route(): def send_response(request: httpx.Request) -> httpx.Response: assert str(request.url) == ( @@ -287,7 +342,7 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_route(): client.close() -def test_azure_gpt_transcribe_preserves_dated_api_version(): +def test_azure_gpt_transcribe_preserves_dated_api_version(local_model_cost_map: None) -> None: resolved_api_version = _validate_gpt_transcription_request( model="gpt-transcribe", custom_llm_provider="azure", @@ -298,3 +353,40 @@ def test_azure_gpt_transcribe_preserves_dated_api_version(): ) assert resolved_api_version == "2025-04-01-preview" + + +@pytest.mark.asyncio +async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + return httpx.Response(200, json={"text": "hello"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncAzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version="2025-04-01-preview", + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = await AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"]}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version="2025-04-01-preview", + client=client, + atranscription=True, + ) + + assert response.text == "hello" + await client.close() diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 5d3276dfae1..c4b92b6e902 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -463,6 +463,29 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) +def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + assert ( + azure_realtime_protocol_for_client( + None, + model="gpt-realtime-2.1", + realtime_mode="realtime", + query_params=None, + websocket=_BETA_CLIENT, + ) + == "GA" + ) + with pytest.raises(ValueError, match="requires the Azure OpenAI v1 Realtime API"): + azure_realtime_protocol_for_client( + "beta", + model="gpt-realtime-2.1", + realtime_mode="realtime", + query_params=None, + websocket=_BETA_CLIENT, + ) + + async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: connect: Final = _ConnectThatStopsAfterCapturingTheUrl() with patch("websockets.connect", connect): From 828707b9f2fda2d204099b85a7dd5fdf980feab5 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 03:09:01 -0500 Subject: [PATCH 12/21] fix(realtime): preserve input usage and provider routing --- .../litellm_core_utils/realtime_streaming.py | 37 ++++++++---- litellm/llms/azure/audio_transcriptions.py | 18 +++++- litellm/llms/azure/realtime/handler.py | 12 +++- litellm/main.py | 33 +++++----- ...odel_prices_and_context_window_backup.json | 3 + litellm/types/llms/openai.py | 2 +- model_prices_and_context_window.json | 3 + .../test_realtime_streaming.py | 30 +++++++++- .../realtime/test_transcription_sessions.py | 7 ++- tests/test_litellm/test_cost_calculator.py | 16 ++++- .../transcriptions/test_gpt_transcribe.py | 60 +++++-------------- tests/unit/realtime_api/test_main.py | 7 ++- 12 files changed, 138 insertions(+), 90 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index cc206079c48..c58a7573fc0 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -440,18 +440,30 @@ class RealTimeStreaming: if event_obj.get("type") == "session.closed": usage: Final = event_obj.get("usage") output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None - if isinstance(output_seconds, (int, float)): - input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None + input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None + synthetic_output_seconds: Final = ( + self._translation_output_audio_bytes / self._translation_output_bytes_per_second + if self._translation_output_audio_bytes > 0 + else None + ) + resolved_output_seconds: Final = ( + output_seconds if isinstance(output_seconds, (int, float)) else synthetic_output_seconds + ) + if isinstance(input_seconds, (int, float)) or resolved_output_seconds is not None: if not self._should_store_message(event_obj): - self.messages.append( - OpenAIRealtimeTranslationClosedEvent( - type="session.closed", - usage=OpenAIRealtimeTranslationDurationUsage( - type="duration", - output_seconds=output_seconds, - **({"input_seconds": input_seconds} if isinstance(input_seconds, (int, float)) else {}), - ), + normalized_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=float(input_seconds), + output_seconds=float(resolved_output_seconds or 0.0), ) + if isinstance(input_seconds, (int, float)) + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=float(resolved_output_seconds or 0.0) + ) + ) + self.messages.append( + OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=normalized_usage) ) self._translation_usage_finalized = True return @@ -500,7 +512,10 @@ class RealTimeStreaming: if event.get("type") != "session.closed": continue event_usage = event.get("usage") # rebind-ok: each close event carries independent usage - if isinstance(event_usage, dict) and isinstance(event_usage.get("output_seconds"), (int, float)): + if isinstance(event_usage, dict) and ( + isinstance(event_usage.get("input_seconds"), (int, float)) + or isinstance(event_usage.get("output_seconds"), (int, float)) + ): self._translation_usage_finalized = True return if self._translation_output_audio_bytes == 0: diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index a6f2b900615..dee8de8b7cb 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from pydantic import BaseModel +import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data @@ -42,6 +43,19 @@ class AzureAudioTranscription(AzureChatCompletion): ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) + model_info: Final = ( + litellm.get_model_info(model=model, custom_llm_provider="azure") + if f"azure/{model}" in litellm.model_cost + else None + ) + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + resolved_api_version: Final = ( + litellm.AZURE_DEFAULT_API_VERSION + if provider_specific_entry is not None + and provider_specific_entry.get("transcription_deployment_api") == 1 + and api_version in ("v1", "latest", "preview") + else api_version + ) if atranscription is True: return self.async_audio_transcriptions( @@ -51,7 +65,7 @@ class AzureAudioTranscription(AzureChatCompletion): timeout=timeout, api_key=api_key, api_base=api_base, - api_version=api_version, + api_version=resolved_api_version, client=client, max_retries=max_retries, logging_obj=logging_obj, @@ -61,7 +75,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) azure_client: Final = self.get_azure_openai_client( - api_version=api_version, + api_version=resolved_api_version, api_base=api_base, api_key=api_key, model=model, diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index d3b7ba6910f..7b38fc85365 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -42,10 +42,16 @@ async def forward_messages(client_ws: Any, backend_ws: Any): def azure_realtime_requires_ga(model: str) -> bool: try: - model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") + azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly - return False - return (model_info.get("provider_specific_entry") or {}).get("realtime_ga_only") == 1 + try: + openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + return False + openai_entry: Final = openai_model_info.get("provider_specific_entry") + return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1 + azure_entry: Final = azure_model_info.get("provider_specific_entry") + return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1 def azure_realtime_protocol_for_client( diff --git a/litellm/main.py b/litellm/main.py index 0f644bcff5a..b00e77d804e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7877,16 +7877,15 @@ def _validate_gpt_transcription_request( language: str | None, languages: Sequence[str] | None, response_format: str | None, - api_version: str | None, -) -> str | None: - model_cost_key: Final = f"{custom_llm_provider}/{model}" if custom_llm_provider == "azure" else model - model_info: Final = ( - get_model_info(model=model, custom_llm_provider=custom_llm_provider) - if model_cost_key in litellm.model_cost - else {} +) -> None: + model_cost_key: Final = next( + (key for key in (f"{custom_llm_provider}/{model}", model) if key in litellm.model_cost), None ) - supported_endpoints: Final = model_info.get("supported_endpoints") - provider_specific_entry: Final = model_info.get("provider_specific_entry") or {} + model_info: Final = ( + get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_cost_key is not None else None + ) + supported_endpoints: Final = model_info.get("supported_endpoints") if model_info is not None else None + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None if language is not None and languages is not None: raise litellm.UnsupportedParamsError( message="language and languages cannot be used together", @@ -7899,17 +7898,16 @@ def _validate_gpt_transcription_request( model=model, llm_provider=custom_llm_provider, ) - if provider_specific_entry.get("transcription_json_only") == 1 and response_format not in (None, "json"): + if ( + provider_specific_entry is not None + and provider_specific_entry.get("transcription_json_only") == 1 + and response_format not in (None, "json") + ): raise litellm.UnsupportedParamsError( message=f"{model} only supports response_format='json'", model=model, llm_provider=custom_llm_provider, ) - if provider_specific_entry.get("transcription_deployment_api") == 1: - if api_version in ("v1", "latest", "preview"): - return litellm.AZURE_DEFAULT_API_VERSION - return api_version - return api_version @client @@ -7976,13 +7974,12 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key - validated_api_version: Final = _validate_gpt_transcription_request( + _validate_gpt_transcription_request( model=model, custom_llm_provider=custom_llm_provider, language=language, languages=languages, response_format=response_format, - api_version=api_version, ) optional_params: Final = get_optional_params_transcription( @@ -8038,7 +8035,7 @@ def transcription( # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - azure_api_version: Final = validated_api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + azure_api_version: Final = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 586cfdc59e5..50ffeb6bf99 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33942,6 +33942,9 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 9c3cf6b25a1..b6fec607951 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2313,7 +2313,7 @@ class OpenAIRealtimeResponseUsage(TypedDict): class OpenAIRealtimeTranslationDurationUsage(TypedDict): type: ReadOnly[Literal["duration"]] - output_seconds: ReadOnly[float] + output_seconds: NotRequired[ReadOnly[float]] input_seconds: NotRequired[ReadOnly[float]] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 586cfdc59e5..50ffeb6bf99 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33942,6 +33942,9 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index f7473787cd1..885a8e52d86 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from dataclasses import dataclass from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -2999,6 +2999,34 @@ def test_translation_prefers_provider_duration_over_audio_byte_estimate(): assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5} +@pytest.mark.parametrize( + ("output_audio_bytes", "expected_usage"), + [ + (0, {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.0}), + (48000, {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0}), + ], +) +def test_translation_preserves_input_only_provider_usage( + output_audio_bytes: int, expected_usage: Mapping[str, str | float] +) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = output_audio_bytes + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == expected_usage + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py index 0c3630bb107..af64d9c7c16 100644 --- a/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py @@ -47,14 +47,15 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version(): @pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) -def test_azure_ga_realtime_http_urls(api_version, monkeypatch: pytest.MonkeyPatch): +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_realtime_http_urls(api_version, model: str, monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) litellm.get_model_info.cache_clear() cfg = AzureRealtimeHTTPConfig() base = "https://my.openai.azure.com" - assert cfg.get_complete_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/client_secrets") - assert cfg.get_realtime_calls_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/calls") + assert cfg.get_complete_url(base, model, api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, model, api_version) == (f"{base}/openai/v1/realtime/calls") assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == ( f"{base}/openai/v1/realtime/transcription_sessions" ) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index bcb0fd15760..31529a72a6e 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4687,12 +4687,22 @@ def test_realtime_translation_duration_cost(_local_model_cost_map): assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) -def test_realtime_translation_duration_cost_includes_provider_input_usage(_local_model_cost_map): +@pytest.mark.parametrize("output_seconds", [None, 2.0]) +def test_realtime_translation_duration_cost_includes_provider_input_usage( + _local_model_cost_map, output_seconds: float | None +): from litellm.cost_calculator import handle_realtime_translation_cost_calculation model: Final = "gpt-realtime-translate" events: Final[OpenAIRealtimeStreamList] = [ - {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 3.0, "output_seconds": 2.0}} + { + "type": "session.closed", + "usage": { + "type": "duration", + "input_seconds": 3.0, + **({"output_seconds": output_seconds} if output_seconds is not None else {}), + }, + } ] actual: Final = handle_realtime_translation_cost_calculation( results=events, @@ -4702,6 +4712,6 @@ def test_realtime_translation_duration_cost_includes_provider_input_usage(_local expected: Final = ( 3 * litellm.model_cost[model]["input_cost_per_second"] - + 2 * litellm.model_cost[model]["output_cost_per_second"] + + (output_seconds or 0) * litellm.model_cost[model]["output_cost_per_second"] ) assert actual == pytest.approx(expected) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 742e1852ea5..26eb6654d87 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -16,7 +16,6 @@ from litellm.llms.openai.transcriptions.gpt_transformation import ( OpenAIGPTTranscribeAudioTranscriptionConfig, ) from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription -from litellm.main import _validate_gpt_transcription_request from litellm.types.utils import TranscriptionResponse from litellm.utils import get_optional_params_transcription @@ -252,7 +251,19 @@ def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: No ) -def test_azure_async_gpt_transcribe_forwards_v1_api_version(): +@pytest.mark.parametrize( + ("api_version", "expected_api_version"), + [ + ("v1", litellm.AZURE_DEFAULT_API_VERSION), + ("latest", litellm.AZURE_DEFAULT_API_VERSION), + ("preview", litellm.AZURE_DEFAULT_API_VERSION), + (None, None), + ("2025-04-01-preview", "2025-04-01-preview"), + ], +) +def test_azure_gpt_transcribe_resolves_api_version_in_provider( + local_model_cost_map: None, api_version: str | None, expected_api_version: str | None +) -> None: handler = AzureAudioTranscription() handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) @@ -266,39 +277,11 @@ def test_azure_async_gpt_transcribe_forwards_v1_api_version(): max_retries=0, api_key="sk-test", api_base="https://example.openai.azure.com", - api_version="v1", + api_version=api_version, atranscription=True, ) - assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == "v1" - - -@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) -def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(local_model_cost_map: None, api_version: str) -> None: - resolved_api_version = _validate_gpt_transcription_request( - model="gpt-transcribe", - custom_llm_provider="azure", - language=None, - languages=None, - response_format="json", - api_version=api_version, - ) - - assert resolved_api_version == litellm.AZURE_DEFAULT_API_VERSION - - -def test_azure_gpt_transcribe_keeps_unset_api_version_for_configured_default(local_model_cost_map: None) -> None: - assert ( - _validate_gpt_transcription_request( - model="gpt-transcribe", - custom_llm_provider="azure", - language=None, - languages=None, - response_format="json", - api_version=None, - ) - is None - ) + assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == expected_api_version def test_azure_gpt_transcribe_uses_deployment_scoped_route(): @@ -342,19 +325,6 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_route(): client.close() -def test_azure_gpt_transcribe_preserves_dated_api_version(local_model_cost_map: None) -> None: - resolved_api_version = _validate_gpt_transcription_request( - model="gpt-transcribe", - custom_llm_provider="azure", - language=None, - languages=None, - response_format="json", - api_version="2025-04-01-preview", - ) - - assert resolved_api_version == "2025-04-01-preview" - - @pytest.mark.asyncio async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): async def send_response(request: httpx.Request) -> httpx.Response: diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index c4b92b6e902..9b0ac1e531d 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -463,13 +463,14 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) -def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map) -> None: +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map, model: str) -> None: from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client assert ( azure_realtime_protocol_for_client( None, - model="gpt-realtime-2.1", + model=model, realtime_mode="realtime", query_params=None, websocket=_BETA_CLIENT, @@ -479,7 +480,7 @@ def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map) with pytest.raises(ValueError, match="requires the Azure OpenAI v1 Realtime API"): azure_realtime_protocol_for_client( "beta", - model="gpt-realtime-2.1", + model=model, realtime_mode="realtime", query_params=None, websocket=_BETA_CLIENT, From 8f23d203670c36386a65e5ed0fe4fe3d0be7cf6e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 03:23:57 -0500 Subject: [PATCH 13/21] fix(realtime): bill retained output and route Azure aliases --- .../litellm_core_utils/realtime_streaming.py | 12 +++++++++- litellm/llms/azure/audio_transcriptions.py | 12 +--------- ...odel_prices_and_context_window_backup.json | 1 - model_prices_and_context_window.json | 1 - .../test_realtime_streaming.py | 22 +++++++++++++++++++ .../transcriptions/test_gpt_transcribe.py | 7 +++--- 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index c58a7573fc0..25fc063a578 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -450,7 +450,17 @@ class RealTimeStreaming: output_seconds if isinstance(output_seconds, (int, float)) else synthetic_output_seconds ) if isinstance(input_seconds, (int, float)) or resolved_output_seconds is not None: - if not self._should_store_message(event_obj): + if self._should_store_message(event_obj): + if not isinstance(output_seconds, (int, float)) and synthetic_output_seconds is not None: + self.messages.append( + OpenAIRealtimeTranslationClosedEvent( + type="session.closed", + usage=OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=synthetic_output_seconds + ), + ) + ) + else: normalized_usage: Final = ( OpenAIRealtimeTranslationDurationUsage( type="duration", diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index dee8de8b7cb..754c319d0d0 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -43,18 +43,8 @@ class AzureAudioTranscription(AzureChatCompletion): ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) - model_info: Final = ( - litellm.get_model_info(model=model, custom_llm_provider="azure") - if f"azure/{model}" in litellm.model_cost - else None - ) - provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None resolved_api_version: Final = ( - litellm.AZURE_DEFAULT_API_VERSION - if provider_specific_entry is not None - and provider_specific_entry.get("transcription_deployment_api") == 1 - and api_version in ("v1", "latest", "preview") - else api_version + litellm.AZURE_DEFAULT_API_VERSION if api_version in ("v1", "latest", "preview") else api_version ) if atranscription is True: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50ffeb6bf99..0b2117a627e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -234,7 +234,6 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, - "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50ffeb6bf99..0b2117a627e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -234,7 +234,6 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, - "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 885a8e52d86..205e18e26fe 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -3027,6 +3027,28 @@ def test_translation_preserves_input_only_provider_usage( assert closed_events[0]["usage"] == expected_usage +def test_translation_retained_input_only_close_event_bills_captured_output(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*") + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + close_event: Final = {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage_events: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert len(usage_events) == 2 + assert sum(usage.get("input_seconds", 0.0) for usage in usage_events) == 0.25 + assert sum(usage.get("output_seconds", 0.0) for usage in usage_events) == 1.0 + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 26eb6654d87..d192550943c 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -261,14 +261,15 @@ def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: No ("2025-04-01-preview", "2025-04-01-preview"), ], ) -def test_azure_gpt_transcribe_resolves_api_version_in_provider( - local_model_cost_map: None, api_version: str | None, expected_api_version: str | None +@pytest.mark.parametrize("model", ["gpt-transcribe", "custom-transcribe-deployment"]) +def test_azure_audio_transcription_resolves_api_version_in_provider( + model: str, api_version: str | None, expected_api_version: str | None ) -> None: handler = AzureAudioTranscription() handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) handler.audio_transcriptions( - model="gpt-transcribe", + model=model, audio_file=io.BytesIO(b"audio"), optional_params={"stream": True}, logging_obj=MagicMock(), From 275bc4c524b12fe3550904b8ba89302d3858ad54 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 03:40:09 -0500 Subject: [PATCH 14/21] fix(realtime): reject malformed provider durations --- litellm/cost_calculator.py | 9 ++++-- .../litellm_core_utils/audio_utils/utils.py | 11 +++++++ .../litellm_core_utils/realtime_streaming.py | 27 +++++++++------- litellm/realtime_api/README.md | 14 +-------- .../test_realtime_streaming.py | 21 +++++++++++++ tests/test_litellm/test_cost_calculator.py | 31 +++++++++++++++++++ 6 files changed, 85 insertions(+), 28 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 98af08f6472..951e336b766 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -18,6 +18,7 @@ from litellm.constants import ( DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, ) +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -3030,12 +3031,14 @@ def handle_realtime_translation_cost_calculation( if isinstance(usage := result.get("usage"), dict) ) input_seconds: Final = sum( - float(usage["input_seconds"]) for usage in usage_events if isinstance(usage.get("input_seconds"), (int, float)) + seconds + for usage in usage_events + if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None ) output_seconds: Final = sum( - float(usage["output_seconds"]) + seconds for usage in usage_events - if isinstance(usage.get("output_seconds"), (int, float)) + if (seconds := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None ) if input_seconds <= 0 and output_seconds <= 0: return 0.0 diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index dab3e48f91a..9fdfce057d9 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -3,6 +3,7 @@ Utils used for litellm.transcription() and litellm.atranscription() """ import hashlib +import math import os from dataclasses import dataclass from typing import Final @@ -17,6 +18,16 @@ from litellm.types.files import ( from litellm.types.utils import FileTypes +def normalized_audio_duration_seconds(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + seconds: Final = float(value) + except OverflowError: + return None + return seconds if math.isfinite(seconds) and seconds >= 0 else None + + @dataclass class ProcessedAudioFile: """ diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 25fc063a578..a8284a20ad7 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly import litellm from litellm._logging import redact_internal_details_from_client_message, verbose_logger from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( @@ -439,19 +440,21 @@ class RealTimeStreaming: return if event_obj.get("type") == "session.closed": usage: Final = event_obj.get("usage") - output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None - input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None + output_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("output_seconds")) if isinstance(usage, dict) else None + ) + input_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("input_seconds")) if isinstance(usage, dict) else None + ) synthetic_output_seconds: Final = ( self._translation_output_audio_bytes / self._translation_output_bytes_per_second if self._translation_output_audio_bytes > 0 else None ) - resolved_output_seconds: Final = ( - output_seconds if isinstance(output_seconds, (int, float)) else synthetic_output_seconds - ) - if isinstance(input_seconds, (int, float)) or resolved_output_seconds is not None: + resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds + if input_seconds is not None or resolved_output_seconds is not None: if self._should_store_message(event_obj): - if not isinstance(output_seconds, (int, float)) and synthetic_output_seconds is not None: + if output_seconds is None and synthetic_output_seconds is not None: self.messages.append( OpenAIRealtimeTranslationClosedEvent( type="session.closed", @@ -464,10 +467,10 @@ class RealTimeStreaming: normalized_usage: Final = ( OpenAIRealtimeTranslationDurationUsage( type="duration", - input_seconds=float(input_seconds), + input_seconds=input_seconds, output_seconds=float(resolved_output_seconds or 0.0), ) - if isinstance(input_seconds, (int, float)) + if input_seconds is not None else OpenAIRealtimeTranslationDurationUsage( type="duration", output_seconds=float(resolved_output_seconds or 0.0) ) @@ -522,9 +525,9 @@ class RealTimeStreaming: if event.get("type") != "session.closed": continue event_usage = event.get("usage") # rebind-ok: each close event carries independent usage - if isinstance(event_usage, dict) and ( - isinstance(event_usage.get("input_seconds"), (int, float)) - or isinstance(event_usage.get("output_seconds"), (int, float)) + if ( + isinstance(event_usage, dict) + and normalized_audio_duration_seconds(event_usage.get("output_seconds")) is not None ): self._translation_usage_finalized = True return diff --git a/litellm/realtime_api/README.md b/litellm/realtime_api/README.md index d9699f57da7..d810de2f24f 100644 --- a/litellm/realtime_api/README.md +++ b/litellm/realtime_api/README.md @@ -6,16 +6,4 @@ Supported endpoints: Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI. -Billing visibility: -- WebSocket sessions pass provider usage events through LiteLLM and support local spend tracking -- Client-secret and SDP call endpoints only proxy session setup; subsequent WebRTC media and usage events travel over the peer connection, so LiteLLM cannot record inference spend or enforce spend-based budgets for those sessions -- Use the proxied WebSocket transport when LiteLLM spend logs and budgets must include Realtime inference - -Non-billable Realtime protocols are disabled by default. Operators who accept the billing and budget-enforcement limitation can opt in: - -```yaml -general_settings: - allow_non_billable_realtime_protocols: true -``` - -For user-facing documentation and usage examples, see the litellm-docs repo. +For user-facing documentation and usage examples, see the litellm-docs repo. \ No newline at end of file diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 205e18e26fe..6dd582e2cec 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -3049,6 +3049,27 @@ def test_translation_retained_input_only_close_event_bills_captured_output(monke assert sum(usage.get("output_seconds", 0.0) for usage in usage_events) == 1.0 +@pytest.mark.parametrize("malformed_output", [-1.0, float("nan"), float("inf"), 10**1000]) +def test_translation_malformed_provider_duration_uses_captured_audio(malformed_output: float | int) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": malformed_output}} + ) + streaming._finalize_translation_usage() + + close_events: Final = tuple(event for event in streaming.messages if event.get("type") == "session.closed") + assert len(close_events) == 1 + assert close_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0} + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 31529a72a6e..cf2e43a2540 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4715,3 +4715,34 @@ def test_realtime_translation_duration_cost_includes_provider_input_usage( + (output_seconds or 0) * litellm.model_cost[model]["output_cost_per_second"] ) assert actual == pytest.approx(expected) + + +@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( + _local_model_cost_map, malformed_duration: float | int | bool +) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + input_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5}, + } + ] + output_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration}, + } + ] + + input_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + output_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + + assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"]) + assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) From 9e1d36489eab36c4d7c9c35ee002623a373299f9 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 04:34:03 -0500 Subject: [PATCH 15/21] fix(realtime): preserve batch cache rates and Azure v1 routes --- litellm/llms/azure/audio_transcriptions.py | 9 +++- ...odel_prices_and_context_window_backup.json | 24 ++++++++++ model_prices_and_context_window.json | 24 ++++++++++ .../llms/azure/test_audio_transcriptions.py | 44 +++++++++++++++++++ .../transcriptions/test_gpt_transcribe.py | 34 -------------- 5 files changed, 100 insertions(+), 35 deletions(-) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 754c319d0d0..e5675d4c82f 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -43,8 +43,15 @@ class AzureAudioTranscription(AzureChatCompletion): ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} sdk_data: Final = sdk_compatible_transcription_request_data(data) + model_info: Final = litellm.model_cost.get(f"azure/{model}") + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + requires_deployment_api: Final = model_info is None or ( + provider_specific_entry is not None and provider_specific_entry.get("transcription_deployment_api") == 1 + ) resolved_api_version: Final = ( - litellm.AZURE_DEFAULT_API_VERSION if api_version in ("v1", "latest", "preview") else api_version + litellm.AZURE_DEFAULT_API_VERSION + if requires_deployment_api and api_version in ("v1", "latest", "preview") + else api_version ) if atranscription is True: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0b2117a627e..82c2d9519b1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -234,6 +234,7 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, + "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", @@ -25728,6 +25729,7 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -25822,6 +25824,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -25905,6 +25908,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -25997,6 +26001,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -26056,6 +26061,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -26112,6 +26118,7 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -26717,6 +26724,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -26776,6 +26784,7 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26833,6 +26842,7 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26891,6 +26901,7 @@ "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28638,6 +28649,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -28697,6 +28709,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28754,6 +28767,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28812,6 +28826,7 @@ "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -33499,6 +33514,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -33679,6 +33695,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -33779,6 +33796,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -48117,6 +48135,7 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -48163,6 +48182,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -48198,6 +48218,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -48290,6 +48311,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -48350,6 +48372,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -48407,6 +48430,7 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0b2117a627e..82c2d9519b1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -234,6 +234,7 @@ "mode": "audio_transcription", "provider_specific_entry": { "realtime_ga_only": 1, + "transcription_deployment_api": 1, "transcription_json_only": 1 }, "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", @@ -25728,6 +25729,7 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -25822,6 +25824,7 @@ }, "gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -25905,6 +25908,7 @@ }, "gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -25997,6 +26001,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -26056,6 +26061,7 @@ "gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -26112,6 +26118,7 @@ }, "deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -26717,6 +26724,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_token": 1.5e-06, "input_cost_per_audio_token": 1.5e-06, "litellm_provider": "vertex_ai", @@ -26776,6 +26784,7 @@ "vertex_ai/gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26833,6 +26842,7 @@ "vertex_ai/gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -26891,6 +26901,7 @@ "vertex_ai/gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28638,6 +28649,7 @@ "prompt_cache_min_tokens": 4096, "deprecation_date": "2027-05-19", "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_batches": 7.5e-08, "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", @@ -28697,6 +28709,7 @@ "gemini-3.6-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28754,6 +28767,7 @@ "gemini-3.7-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -28812,6 +28826,7 @@ "gemini-3.8-flash": { "prompt_cache_min_tokens": 4096, "cache_read_input_token_cost": 7.5e-08, + "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, "input_cost_per_token": 7.5e-07, "input_cost_per_token_batches": 3.75e-07, @@ -33499,6 +33514,7 @@ }, "gpt-5-2025-08-07": { "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_batches": 6.25e-08, "cache_read_input_token_cost_flex": 6.25e-08, "cache_read_input_token_cost_priority": 2.5e-07, "deprecation_date": "2026-12-11", @@ -33679,6 +33695,7 @@ }, "gpt-5-mini-2025-08-07": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "deprecation_date": "2026-12-11", @@ -33779,6 +33796,7 @@ }, "gpt-5-nano-2025-08-07": { "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_batches": 2.5e-09, "cache_read_input_token_cost_flex": 2.5e-09, "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, @@ -48117,6 +48135,7 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 1e-07, @@ -48163,6 +48182,7 @@ }, "vertex_ai/gemini-3.1-flash-image": { "cache_read_input_token_cost": 5e-08, + "cache_read_input_token_cost_batches": 2.5e-08, "cache_read_input_token_cost_flex": 2.5e-08, "deprecation_date": "2027-05-28", "input_cost_per_image": 0.00056, @@ -48198,6 +48218,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "input_cost_per_image": 0.00028, "input_cost_per_token": 2.5e-07, @@ -48290,6 +48311,7 @@ "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, "cache_read_input_token_cost_priority": 4.5e-08, "input_cost_per_audio_token": 5e-07, @@ -48350,6 +48372,7 @@ "vertex_ai/gemini-3.5-flash-lite": { "deprecation_date": "2027-07-21", "cache_read_input_token_cost": 3e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 1.5e-08, "cache_read_input_token_cost_priority": 5.4e-08, "input_cost_per_token": 3e-07, @@ -48407,6 +48430,7 @@ }, "vertex_ai/deep-research-pro-preview-12-2025": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, diff --git a/tests/test_litellm/llms/azure/test_audio_transcriptions.py b/tests/test_litellm/llms/azure/test_audio_transcriptions.py index 4f1906d80be..267ddaf9a5d 100644 --- a/tests/test_litellm/llms/azure/test_audio_transcriptions.py +++ b/tests/test_litellm/llms/azure/test_audio_transcriptions.py @@ -1,6 +1,8 @@ +import io import json from pathlib import Path from typing import Final +from unittest.mock import MagicMock import httpx import pytest @@ -9,6 +11,8 @@ from openai import AzureOpenAI import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.types.utils import TranscriptionResponse AUDIO_FILE: Final = Path(__file__).parents[3] / "gettysburg.wav" WHISPER_COST_PER_SECOND: Final = 0.0001 @@ -39,3 +43,43 @@ def test_azure_transcription_keeps_the_azure_provider(): assert response._hidden_params["custom_llm_provider"] == "azure" assert json.loads(response.model_dump_json())["text"] == "Four score and seven years ago" + + +@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) +@pytest.mark.parametrize( + ("model", "expected_path"), + [ + ("whisper-1", "/openai/v1/audio/transcriptions"), + ("gpt-transcribe", "/openai/deployments/gpt-transcribe/audio/transcriptions"), + ("custom-transcribe-deployment", "/openai/deployments/custom-transcribe-deployment/audio/transcriptions"), + ], +) +def test_azure_transcription_alias_uses_model_route( + monkeypatch: pytest.MonkeyPatch, model: str, expected_path: str, api_version: str +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == expected_path + assert request.url.params.get("api-version") == ( + None if expected_path.startswith("/openai/v1/") else litellm.AZURE_DEFAULT_API_VERSION + ) + return httpx.Response(200, json={"text": "hello"}) + + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with httpx.Client(transport=httpx.MockTransport(send_response)) as http_client: + monkeypatch.setattr(litellm, "client_session", http_client) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + response: Final = AzureAudioTranscription().audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="test-key", + api_base="https://example.openai.azure.com", + api_version=api_version, + ) + + assert response.text == "hello" diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index d192550943c..21660fca9f9 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -251,40 +251,6 @@ def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: No ) -@pytest.mark.parametrize( - ("api_version", "expected_api_version"), - [ - ("v1", litellm.AZURE_DEFAULT_API_VERSION), - ("latest", litellm.AZURE_DEFAULT_API_VERSION), - ("preview", litellm.AZURE_DEFAULT_API_VERSION), - (None, None), - ("2025-04-01-preview", "2025-04-01-preview"), - ], -) -@pytest.mark.parametrize("model", ["gpt-transcribe", "custom-transcribe-deployment"]) -def test_azure_audio_transcription_resolves_api_version_in_provider( - model: str, api_version: str | None, expected_api_version: str | None -) -> None: - handler = AzureAudioTranscription() - handler.async_audio_transcriptions = MagicMock(return_value=MagicMock()) - - handler.audio_transcriptions( - model=model, - audio_file=io.BytesIO(b"audio"), - optional_params={"stream": True}, - logging_obj=MagicMock(), - model_response=TranscriptionResponse(), - timeout=10, - max_retries=0, - api_key="sk-test", - api_base="https://example.openai.azure.com", - api_version=api_version, - atranscription=True, - ) - - assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == expected_api_version - - def test_azure_gpt_transcribe_uses_deployment_scoped_route(): def send_response(request: httpx.Request) -> httpx.Response: assert str(request.url) == ( From 6a3c36fcc344d4848d72bef0809741b2e4b7bd80 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 04:49:54 -0500 Subject: [PATCH 16/21] fix(transcription): reject nonfinite provider durations --- litellm/cost_calculator.py | 4 +--- tests/test_litellm/test_cost_calculator.py | 9 +++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 951e336b766..87a5c6c867d 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1038,9 +1038,7 @@ def _get_transcription_usage_duration(completion_response: object) -> float | No seconds: Final = ( usage_object.get("seconds") if isinstance(usage_object, dict) else getattr(usage_object, "seconds", None) ) - if isinstance(seconds, bool) or not isinstance(seconds, (int, float)) or seconds < 0: - return None - return float(seconds) + return normalized_audio_duration_seconds(seconds) def _is_known_usage_objects(usage_obj): diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index cf2e43a2540..45423c78b21 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -334,6 +334,15 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type(): assert _transcription_usage_cost({}, {}) == 0.0 +@pytest.mark.parametrize("malformed_seconds", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_transcription_ignores_invalid_provider_duration(malformed_seconds: float | int | bool) -> None: + from litellm.cost_calculator import _get_transcription_usage_duration + + response: Final = SimpleNamespace(usage={"type": "duration", "seconds": malformed_seconds}) + + assert _get_transcription_usage_duration(response) is None + + def test_get_transcription_model_falls_back_to_session_model(monkeypatch): """session.model is used when transcription-specific model fields are absent.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") From 152b1e15d2c05737341b5c5ceccf6522508d9932 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:03:44 -0500 Subject: [PATCH 17/21] refactor(realtime): align annotations with current lint rules --- litellm/litellm_core_utils/realtime_streaming.py | 3 +-- litellm/llms/custom_httpx/llm_http_handler.py | 2 +- .../llms/openai/transcriptions/gpt_transformation.py | 4 +--- litellm/llms/openai/transcriptions/handler.py | 4 +++- litellm/proxy/realtime_endpoints/endpoints.py | 10 +++------- litellm/realtime_api/main.py | 10 ++++++---- 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a8284a20ad7..455967f7cfe 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -524,9 +524,8 @@ class RealTimeStreaming: for event in self.messages: if event.get("type") != "session.closed": continue - event_usage = event.get("usage") # rebind-ok: each close event carries independent usage if ( - isinstance(event_usage, dict) + isinstance(event_usage := event.get("usage"), dict) and normalized_audio_duration_seconds(event_usage.get("output_seconds")) is not None ): self._translation_usage_finalized = True diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 136926f2912..928ba17d305 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6162,7 +6162,7 @@ class BaseLLMHTTPHandler: configured_client: Final = openai_client.with_options( timeout=timeout, set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping - key: str(value) # mutable-ok: SDK headers are materialized as a concrete string mapping + key: str(value) for key, value in (extra_headers or {}).items() # mutable-ok: SDK requires concrete headers }, ) diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index 81f3abd217a..c55ecf1e56c 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -40,9 +40,7 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig): - def get_supported_openai_params( # mutable-ok: base transcription interface returns a mutable parameter list - self, model: str - ) -> list[OpenAIAudioTranscriptionOptionalParams]: + def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: return [ "prompt", "response_format", diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index de020f4cae3..ac57cc72290 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -57,7 +57,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): if data.get("stream") is True: stream_response: Final = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) return None, stream_response - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**sdk_data, timeout=timeout) + raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( + **sdk_data, timeout=timeout + ) headers: Final = dict(raw_response.headers) response: Final = raw_response.parse() return headers, response diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index eb3a8430923..f358111835b 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -139,7 +139,7 @@ async def _authorize_and_bind_nested_transcription_models( session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization user_api_key_dict: UserAPIKeyAuth, llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list - llm_router: Any, + llm_router: "Router | None", ) -> None: nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data)) for nested_model in nested_models: @@ -515,18 +515,14 @@ async def proxy_realtime_calls( session_type = _coerce_realtime_session_type(raw_session_type) if is_translation_request != (raw_session_type == "translation"): return Response( - content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping - {"error": "Token is not valid for this Realtime endpoint"} - ), + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), status_code=http_status.HTTP_401_UNAUTHORIZED, media_type="application/json", ) else: if is_translation_request: return Response( - content=json.dumps( # mutable-ok: JSON encoder requires the endpoint error payload mapping - {"error": "Token is not valid for this Realtime endpoint"} - ), + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), status_code=http_status.HTTP_401_UNAUTHORIZED, media_type="application/json", ) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 05a41bf5844..9ac86177fd7 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -75,12 +75,14 @@ def _with_resolved_session_model(session: dict[str, object], model_name: str) -> audio = session.get("audio") audio = audio if isinstance(audio, dict) else {} # mutable-ok: nested session model is rebuilt locally audio_input = audio.get("input") - audio_input = ( # mutable-ok: nested session model is rebuilt locally - audio_input if isinstance(audio_input, dict) else {} + audio_input = ( + audio_input if isinstance(audio_input, dict) else {} # mutable-ok: nested session model is rebuilt locally ) transcription = audio_input.get("transcription") - transcription = ( # mutable-ok: nested session model is rebuilt locally - transcription if isinstance(transcription, dict) else {} + transcription = ( + transcription + if isinstance(transcription, dict) + else {} # mutable-ok: nested session model is rebuilt locally ) return { # mutable-ok: provider routing requires an independently mutable session payload **session, From a66dd7dda03533d0c4843ea86e3da4e048d520fa Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:24:14 -0500 Subject: [PATCH 18/21] fix(realtime): preserve translation deployment pricing and input usage --- litellm/cost_calculator.py | 36 +++++---- .../litellm_core_utils/realtime_streaming.py | 77 +++++++++++++++---- tests/unit/cookbook/__init__.py | 0 .../test_realtime_streaming.py | 56 ++++++++++++++ tests/unit/test_cost_calculator.py | 49 ++++++++++++ 5 files changed, 188 insertions(+), 30 deletions(-) create mode 100644 tests/unit/cookbook/__init__.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 01a84ac9c9a..81d913a24b9 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -3014,6 +3014,7 @@ def handle_realtime_stream_cost_calculation( results=results, custom_llm_provider=custom_llm_provider, litellm_model_name=litellm_model_name, + potential_model_names=potential_model_names, ) total_cost: Final = input_cost_per_token + output_cost_per_token + transcription_cost + translation_cost @@ -3043,6 +3044,7 @@ def handle_realtime_translation_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, litellm_model_name: str, + potential_model_names: Sequence[str | None] = (), ) -> float: usage_events: Final = tuple( usage @@ -3062,22 +3064,28 @@ def handle_realtime_translation_cost_calculation( ) if input_seconds <= 0 and output_seconds <= 0: return 0.0 - try: - model_info: Final = litellm.get_model_info( - model=litellm_model_name, - custom_llm_provider=custom_llm_provider, - ) - except Exception: # noqa: BLE001 # unknown model metadata should yield zero translation cost - return 0.0 - input_cost_per_second: Final = model_info.get("input_cost_per_second") - output_cost_per_second: Final = model_info.get("output_cost_per_second") - input_cost: Final = ( - input_seconds * input_cost_per_second if isinstance(input_cost_per_second, (int, float)) else 0.0 + model_infos: Final = tuple( + _get_model_info_or_none(model, custom_llm_provider) + for model in (*potential_model_names, litellm_model_name) + if model is not None ) - output_cost: Final = ( - output_seconds * output_cost_per_second if isinstance(output_cost_per_second, (int, float)) else 0.0 + input_cost_per_second: Final = next( + ( + rate + for info in model_infos + if (rate := _declared_transcription_rate(info, ("input_cost_per_second",))) is not None + ), + 0.0, ) - return input_cost + output_cost + output_cost_per_second: Final = next( + ( + rate + for info in model_infos + if (rate := _declared_transcription_rate(info, ("output_cost_per_second",))) is not None + ), + 0.0, + ) + return input_seconds * input_cost_per_second + output_seconds * output_cost_per_second def handle_realtime_transcription_cost_calculation( diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 455967f7cfe..95d6b1eb7ba 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -154,6 +154,8 @@ class RealTimeStreaming: self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] self._is_translation_session = translation_session + self._translation_input_seconds = 0.0 + self._translation_input_bytes_per_second = 48000.0 self._translation_output_audio_bytes = 0 self._translation_output_bytes_per_second = 48000.0 self._translation_usage_finalized = False @@ -496,27 +498,61 @@ class RealTimeStreaming: return self._translation_output_audio_bytes += len(decoded) + @staticmethod + def _translation_audio_bytes_per_second(audio_format: object) -> float | None: + if audio_format == "pcm16": + return 48000.0 + if audio_format in ("g711_ulaw", "g711_alaw"): + return 8000.0 + if not isinstance(audio_format, Mapping): + return None + rate: Final = normalized_audio_duration_seconds(audio_format.get("rate")) + if rate is None or rate <= 0: + return None + match audio_format.get("type"): + case "audio/pcm": + return rate * 2 + case "audio/pcmu" | "audio/pcma": + return rate + case _: + return None + + def _capture_translation_input_audio(self, message: str) -> None: + if not self._is_translation_session: + return + try: + event: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return + self._capture_translation_output_format(event) + if event.get("type") not in ( + "input_audio_buffer.append", + "session.input_audio_buffer.append", + ) or not isinstance(audio := event.get("audio"), str): + return + try: + decoded: Final = base64.b64decode(audio, validate=True) + except (ValueError, TypeError): + return + self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second + def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: session: Final = event_obj.get("session") if not isinstance(session, dict): return audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + input_format: Final = ( + audio_input.get("format") if isinstance(audio_input, dict) else session.get("input_audio_format") + ) + input_rate: Final = self._translation_audio_bytes_per_second(input_format) + if input_rate is not None: + self._translation_input_bytes_per_second = input_rate output: Final = audio.get("output") if isinstance(audio, dict) else None audio_format: Final = output.get("format") if isinstance(output, dict) else None - if isinstance(audio_format, str): - if audio_format in ("g711_ulaw", "g711_alaw"): - self._translation_output_bytes_per_second = 8000.0 - return - if not isinstance(audio_format, dict): - return - format_type: Final = audio_format.get("type") - rate: Final = audio_format.get("rate") - if not isinstance(rate, (int, float)) or rate <= 0: - return - if format_type == "audio/pcm": - self._translation_output_bytes_per_second = float(rate) * 2 - elif format_type in ("audio/pcmu", "audio/pcma"): - self._translation_output_bytes_per_second = float(rate) + output_rate: Final = self._translation_audio_bytes_per_second(audio_format) + if output_rate is not None: + self._translation_output_bytes_per_second = output_rate def _finalize_translation_usage(self) -> None: if self._translation_usage_finalized: @@ -530,10 +566,16 @@ class RealTimeStreaming: ): self._translation_usage_finalized = True return - if self._translation_output_audio_bytes == 0: + if self._translation_output_audio_bytes == 0 and self._translation_input_seconds == 0: return output_seconds: Final = self._translation_output_audio_bytes / self._translation_output_bytes_per_second - synthetic_usage: Final = OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + synthetic_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", input_seconds=self._translation_input_seconds, output_seconds=output_seconds + ) + if self._translation_input_seconds > 0 + else OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + ) self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) self._translation_usage_finalized = True @@ -587,8 +629,11 @@ class RealTimeStreaming: if is_content_message: self._content_sent_after_setup = True sent = True + if sent: + self._capture_translation_input_audio(message) return sent await self.backend_ws.send(message) + self._capture_translation_input_audio(message) return True async def _apply_nested_transcription_model_policy(self, message: str) -> str: diff --git a/tests/unit/cookbook/__init__.py b/tests/unit/cookbook/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 6dd582e2cec..d0ee76bb3c8 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -2933,6 +2933,62 @@ def test_translation_audio_duration_is_finalized_once(event_type: str): assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 1.0} +@pytest.mark.asyncio +@pytest.mark.parametrize("output_bytes", (0, 48000)) +@pytest.mark.parametrize("event_type", ("input_audio_buffer.append", "session.input_audio_buffer.append")) +@pytest.mark.parametrize( + "audio_format,bytes_per_second", + [("pcm16", 48000), ("g711_ulaw", 8000), ({"type": "audio/pcm", "rate": 16000}, 32000)], +) +async def test_translation_disconnect_bills_sent_input_audio( + output_bytes: int, event_type: str, audio_format: str | Mapping[str, object], bytes_per_second: int +) -> None: + import base64 + + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), + backend_ws=backend, + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + await streaming._send_to_backend( + json.dumps({"type": "session.update", "session": {"audio": {"input": {"format": audio_format}}}}) + ) + await streaming._send_to_backend( + json.dumps({"type": event_type, "audio": base64.b64encode(bytes(2 * bytes_per_second)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} + ) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + assert streaming.messages == [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": output_bytes / 48000}, + } + ] + + +@pytest.mark.asyncio +async def test_translation_failed_audio_send_is_not_billed() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock(side_effect=RuntimeError("send failed")) + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + + with pytest.raises(RuntimeError, match="send failed"): + await streaming._send_to_backend(json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})) + streaming._finalize_translation_usage() + + assert streaming.messages == [] + + def test_translation_audio_duration_uses_session_output_format(): import base64 diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 71c2ce3c742..bfb89f98347 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -4967,6 +4967,55 @@ def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: assert actual == pytest.approx(expected) +@pytest.mark.parametrize("input_override,output_override", [(None, None), (0.25, 0.75), (0.0, 0.0), (0.0, None)]) +def test_realtime_translation_uses_deployment_rates_before_base_rates( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + input_override: float | None, + output_override: float | None, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "translation-base", + { + "litellm_provider": "azure", + "mode": "realtime", + "input_cost_per_second": 0.5, + "output_cost_per_second": 1.0, + }, + ) + monkeypatch.setitem( + litellm.model_cost, + "translation-deployment", + { + "litellm_provider": "azure", + "mode": "realtime", + **{ + key: rate + for key, rate in (("input_cost_per_second", input_override), ("output_cost_per_second", output_override)) + if rate is not None + }, + }, + ) + litellm.get_model_info.cache_clear() + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 3.0, "output_seconds": 2.0}} + ] + cost: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=Usage(), + custom_llm_provider="azure", + litellm_model_name="unmapped-provider-deployment", + custom_pricing_model="translation-deployment", + base_pricing_model="translation-base", + ) + + assert cost == pytest.approx( + 3 * (0.5 if input_override is None else input_override) + + 2 * (1.0 if output_override is None else output_override) + ) + + def test_realtime_translation_duration_cost(_local_model_cost_map): from litellm.cost_calculator import handle_realtime_translation_cost_calculation From c0b814dbe839b86410b2335b859a992bb8b0d6f5 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:29:37 -0500 Subject: [PATCH 19/21] test(transcription): verify synchronous stream finalization --- .../transcriptions/test_gpt_transcribe.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py index 21660fca9f9..535b52a17ee 100644 --- a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -3,11 +3,12 @@ import json import wave from collections.abc import Iterator from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest -from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI import litellm from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream @@ -66,6 +67,39 @@ def test_transcription_response_preserves_empty_languages(): assert response.model_dump()["languages"] == [] +def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + 'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + 'data: {"type":"transcript.text.done","text":"hello world",' + '"usage":{"type":"duration","seconds":2.5}}\n\n' + ), + ) + + logging_obj: Final = MagicMock() + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + stream: Final = client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + wrapped: Final = wrap_transcription_stream(stream, logging_obj, datetime(2026, 1, 1)) + received: Final = tuple(wrapped) + wrapped.close() + + assert tuple(event.type for event in received) == ("transcript.text.delta", "transcript.text.done") + logging_obj.success_handler.assert_called_once() + logged_response: Final = logging_obj.success_handler.call_args.args[0] + assert logged_response.text == "hello world" + assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5} + logging_obj.failure_handler.assert_not_called() + + @pytest.mark.asyncio async def test_openai_handler_returns_native_typed_stream(): async def send_response(request: httpx.Request) -> httpx.Response: From 998bde096306eb2ac2fd7a96e046958bba9516a3 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 09:36:33 -0500 Subject: [PATCH 20/21] fix(realtime): retain missing translation input usage at close --- .../litellm_core_utils/realtime_streaming.py | 73 ++++++++----------- .../test_realtime_streaming.py | 48 +++++++++--- 2 files changed, 69 insertions(+), 52 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 95d6b1eb7ba..4d2d4dde999 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -155,7 +155,6 @@ class RealTimeStreaming: self.tool_calls: list[dict] = [] self._is_translation_session = translation_session self._translation_input_seconds = 0.0 - self._translation_input_bytes_per_second = 48000.0 self._translation_output_audio_bytes = 0 self._translation_output_bytes_per_second = 48000.0 self._translation_usage_finalized = False @@ -445,9 +444,14 @@ class RealTimeStreaming: output_seconds: Final = ( normalized_audio_duration_seconds(usage.get("output_seconds")) if isinstance(usage, dict) else None ) - input_seconds: Final = ( + reported_input_seconds: Final = ( normalized_audio_duration_seconds(usage.get("input_seconds")) if isinstance(usage, dict) else None ) + input_seconds: Final = ( + reported_input_seconds + if reported_input_seconds is not None + else self._translation_input_seconds or None + ) synthetic_output_seconds: Final = ( self._translation_output_audio_bytes / self._translation_output_bytes_per_second if self._translation_output_audio_bytes > 0 @@ -456,13 +460,19 @@ class RealTimeStreaming: resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds if input_seconds is not None or resolved_output_seconds is not None: if self._should_store_message(event_obj): - if output_seconds is None and synthetic_output_seconds is not None: + supplemental_usage: Final = OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=float(input_seconds or 0.0) if reported_input_seconds is None else 0.0, + output_seconds=float(synthetic_output_seconds or 0.0) if output_seconds is None else 0.0, + ) + if ( + supplemental_usage.get("input_seconds", 0.0) > 0 + or supplemental_usage.get("output_seconds", 0.0) > 0 + ): self.messages.append( OpenAIRealtimeTranslationClosedEvent( type="session.closed", - usage=OpenAIRealtimeTranslationDurationUsage( - type="duration", output_seconds=synthetic_output_seconds - ), + usage=supplemental_usage, ) ) else: @@ -498,25 +508,6 @@ class RealTimeStreaming: return self._translation_output_audio_bytes += len(decoded) - @staticmethod - def _translation_audio_bytes_per_second(audio_format: object) -> float | None: - if audio_format == "pcm16": - return 48000.0 - if audio_format in ("g711_ulaw", "g711_alaw"): - return 8000.0 - if not isinstance(audio_format, Mapping): - return None - rate: Final = normalized_audio_duration_seconds(audio_format.get("rate")) - if rate is None or rate <= 0: - return None - match audio_format.get("type"): - case "audio/pcm": - return rate * 2 - case "audio/pcmu" | "audio/pcma": - return rate - case _: - return None - def _capture_translation_input_audio(self, message: str) -> None: if not self._is_translation_session: return @@ -524,35 +515,35 @@ class RealTimeStreaming: event: Final = _decode_json_object(message) except (json.JSONDecodeError, TypeError): return - self._capture_translation_output_format(event) - if event.get("type") not in ( - "input_audio_buffer.append", - "session.input_audio_buffer.append", - ) or not isinstance(audio := event.get("audio"), str): + if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str): return try: decoded: Final = base64.b64decode(audio, validate=True) except (ValueError, TypeError): return - self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second + self._translation_input_seconds += len(decoded) / 48000.0 def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: session: Final = event_obj.get("session") if not isinstance(session, dict): return audio: Final = session.get("audio") - audio_input: Final = audio.get("input") if isinstance(audio, dict) else None - input_format: Final = ( - audio_input.get("format") if isinstance(audio_input, dict) else session.get("input_audio_format") - ) - input_rate: Final = self._translation_audio_bytes_per_second(input_format) - if input_rate is not None: - self._translation_input_bytes_per_second = input_rate output: Final = audio.get("output") if isinstance(audio, dict) else None audio_format: Final = output.get("format") if isinstance(output, dict) else None - output_rate: Final = self._translation_audio_bytes_per_second(audio_format) - if output_rate is not None: - self._translation_output_bytes_per_second = output_rate + if isinstance(audio_format, str): + if audio_format in ("g711_ulaw", "g711_alaw"): + self._translation_output_bytes_per_second = 8000.0 + return + if not isinstance(audio_format, dict): + return + format_type: Final = audio_format.get("type") + rate: Final = audio_format.get("rate") + if not isinstance(rate, (int, float)) or rate <= 0: + return + if format_type == "audio/pcm": + self._translation_output_bytes_per_second = float(rate) * 2 + elif format_type in ("audio/pcmu", "audio/pcma"): + self._translation_output_bytes_per_second = float(rate) def _finalize_translation_usage(self) -> None: if self._translation_usage_finalized: diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index d0ee76bb3c8..94b0d215bf4 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -2935,13 +2935,8 @@ def test_translation_audio_duration_is_finalized_once(event_type: str): @pytest.mark.asyncio @pytest.mark.parametrize("output_bytes", (0, 48000)) -@pytest.mark.parametrize("event_type", ("input_audio_buffer.append", "session.input_audio_buffer.append")) -@pytest.mark.parametrize( - "audio_format,bytes_per_second", - [("pcm16", 48000), ("g711_ulaw", 8000), ({"type": "audio/pcm", "rate": 16000}, 32000)], -) async def test_translation_disconnect_bills_sent_input_audio( - output_bytes: int, event_type: str, audio_format: str | Mapping[str, object], bytes_per_second: int + output_bytes: int, ) -> None: import base64 @@ -2955,10 +2950,7 @@ async def test_translation_disconnect_bills_sent_input_audio( translation_session=True, ) await streaming._send_to_backend( - json.dumps({"type": "session.update", "session": {"audio": {"input": {"format": audio_format}}}}) - ) - await streaming._send_to_backend( - json.dumps({"type": event_type, "audio": base64.b64encode(bytes(2 * bytes_per_second)).decode()}) + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) ) streaming._capture_translation_output_audio( {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} @@ -2983,12 +2975,46 @@ async def test_translation_failed_audio_send_is_not_billed() -> None: ) with pytest.raises(RuntimeError, match="send failed"): - await streaming._send_to_backend(json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})) + await streaming._send_to_backend(json.dumps({"type": "session.input_audio_buffer.append", "audio": "AAAA"})) streaming._finalize_translation_usage() assert streaming.messages == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("retain_close", (False, True)) +@pytest.mark.parametrize("reported_input,expected_input", [(None, 2.0), (0.0, 0.0), (0.25, 0.25)]) +async def test_translation_terminal_usage_fills_only_missing_input_duration( + monkeypatch: pytest.MonkeyPatch, retain_close: bool, reported_input: float | None, expected_input: float +) -> None: + import base64 + + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*" if retain_close else None) + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) + ) + close_event: Final = { + "type": "session.closed", + "usage": { + "type": "duration", + "output_seconds": 0.5, + **({"input_seconds": reported_input} if reported_input is not None else {}), + }, + } + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert sum(item.get("input_seconds") or 0.0 for item in usage) == expected_input + assert sum(item.get("output_seconds") or 0.0 for item in usage) == 0.5 + + def test_translation_audio_duration_uses_session_output_format(): import base64 From 6cb6811e45ac67c01ac8a1009c2184b67e2a286f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 10:01:14 -0500 Subject: [PATCH 21/21] fix(transcription): reject streams that bypass output guardrails --- litellm/proxy/proxy_server.py | 6 +++ litellm/proxy/utils.py | 11 ++++ .../proxy/proxy_server/test_routes_audio.py | 51 +++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 26e9b02aace..caf9250c4f6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12423,6 +12423,12 @@ async def audio_transcriptions( call_type="transcription", ) + if data.get("stream") is True and ProxyLogging.has_post_call_guardrails(data, llm_router): + raise HTTPException( + detail="Streaming transcription does not support output guardrails. Use stream=false.", + status_code=status.HTTP_400_BAD_REQUEST, + ) + ## ROUTE TO CORRECT ENDPOINT ## llm_call: Final = await route_request( data=data, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b8cc30ad8a7..d6978d04590 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2874,6 +2874,17 @@ class ProxyLogging: def has_post_call_response_headers_callbacks() -> bool: return ProxyLogging._callback_capabilities().has_post_call_response_headers + @staticmethod + def has_post_call_guardrails(request_data: Mapping[str, object], llm_router: Router | None) -> bool: + guardrail_data: Final = _check_and_merge_model_level_guardrails( + data=dict(request_data), llm_router=llm_router, trust_client_model_info=False + ) + guardrails, _ = _partition_post_call_callbacks() + return bool(pipeline_managed_guardrail_names(guardrail_data, "post_call")) or any( + guardrail.should_run_guardrail(data=guardrail_data, event_type=GuardrailEventHooks.post_call) + for guardrail in guardrails + ) + @staticmethod def has_streaming_callbacks() -> bool: caps: Final = ProxyLogging._callback_capabilities() diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index a756454405e..03cfe3fa67e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -10,13 +10,21 @@ Pins (PR2): from __future__ import annotations import io +from collections.abc import Callable +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi.testclient import TestClient +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep @pytest.fixture @@ -305,3 +313,46 @@ def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcr assert response.headers["content-type"].startswith("text/event-stream") assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\n' assert patched_transcription_stream.closed is True + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize( + "configuration,expected_status", + [("default", 400), ("model", 400), ("policy", 400), ("pre_call", 200), ("disabled", 200)], +) +def test_streaming_transcription_rejects_applicable_output_guardrails( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + configuration: str, + expected_status: int, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="transcription-output", + event_hook=GuardrailEventHooks.pre_call if configuration == "pre_call" else GuardrailEventHooks.post_call, + default_on=configuration in ("default", "pre_call"), + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + proxy_server.llm_router.get_model_list.return_value = ( + [{"litellm_params": {"guardrails": ["transcription-output"]}}] if configuration == "model" else [] + ) + if configuration == "policy": + pipeline: Final = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="transcription-output")]) + proxy_server.proxy_logging_obj.pre_call_hook.side_effect = lambda **kwargs: { + **kwargs["data"], + "metadata": {"_guardrail_pipelines": [("transcription-policy", pipeline)]}, + } + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == expected_status + if expected_status == 400: + assert "stream=false" in response.json()["error"]["message"] + assert "hello world" not in response.text + else: + assert '"text":"hello world"' in response.text