diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py new file mode 100644 index 00000000000..bae89b829ca --- /dev/null +++ b/cookbook/gpt_realtime_translate.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import base64 +import json +import os +import sys +import wave +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode, urlsplit, urlunsplit + +import websockets +from websockets.asyncio.client import ClientConnection + +SAMPLE_RATE = 24_000 +CHANNELS = 1 +SAMPLE_WIDTH = 2 +CHUNK_DURATION_SECONDS = 0.1 +CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS) +OUTPUT_IDLE_TIMEOUT_SECONDS = 3.0 +INITIAL_RESPONSE_TIMEOUT_SECONDS = 30.0 +AUDIO_EVENT_TYPES = frozenset( + { + "session.output_audio.delta", + "response.audio.delta", + "response.output_audio.delta", + } +) +TRANSCRIPT_EVENT_TYPES = frozenset( + { + "session.output_transcript.delta", + "response.text.delta", + "response.output_audio_transcript.delta", + } +) + + +@dataclass(frozen=True, slots=True) +class Settings: + input_wav: Path + output_wav: Path + base_url: str + model: str + target_language: str + trailing_silence_seconds: float + api_key: str + + +def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None: + sys.stdout.write(f"{message}{end}") + if flush: + sys.stdout.flush() + + +def write_stderr(message: str) -> None: + sys.stderr.write(f"{message}\n") + + +def parse_args(argv: Sequence[str] | None = None) -> Settings | str: + parser = argparse.ArgumentParser( + description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio", + ) + parser.add_argument("input_wav", type=Path) + parser.add_argument("--output", type=Path, default=Path("translated.wav")) + parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000")) + parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate")) + parser.add_argument("--target-language", default="fr") + parser.add_argument("--trailing-silence", type=float, default=1.5) + parsed = parser.parse_args(argv) + api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY") + if not api_key: + return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script" + if parsed.trailing_silence < 0: + return "--trailing-silence must be zero or greater" + return Settings( + input_wav=parsed.input_wav, + output_wav=parsed.output, + base_url=parsed.base_url, + model=parsed.model, + target_language=parsed.target_language, + trailing_silence_seconds=parsed.trailing_silence, + api_key=api_key, + ) + + +def translation_url(base_url: str, model: str) -> str | None: + parsed = urlsplit(base_url.rstrip("/")) + scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme) + if not scheme or not parsed.netloc: + return None + base_path = parsed.path.rstrip("/") + realtime_path = ( + f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations" + ) + return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), "")) + + +def read_pcm16_wav(path: Path) -> bytes | str: + try: + with wave.open(str(path), "rb") as source: + actual_format = ( + source.getnchannels(), + source.getsampwidth(), + source.getframerate(), + source.getcomptype(), + ) + expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE") + if actual_format != expected_format: + return ( + f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received " + f"channels={actual_format[0]}, sample_width={actual_format[1]}, " + f"sample_rate={actual_format[2]}, compression={actual_format[3]}" + ) + return source.readframes(source.getnframes()) + except (OSError, EOFError, wave.Error) as exc: + return f"Unable to read {path}: {exc}" + + +def audio_chunks(audio: bytes) -> Iterator[bytes]: + return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES)) + + +def audio_message(audio: bytes) -> str: + return json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(audio).decode("ascii"), + } + ) + + +async def configure_session(connection: ClientConnection, target_language: str) -> str | None: + await connection.send( + json.dumps( + { + "type": "session.update", + "session": {"audio": {"output": {"language": target_language}}}, + } + ) + ) + while True: + raw_event = await asyncio.wait_for(connection.recv(), timeout=20) + event = json.loads(raw_event) + event_type = event.get("type") + if event_type == "session.created": + write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}") + if event_type == "session.updated": + return None + if event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + + +async def send_audio( + connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event +) -> None: + silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + try: + for chunk in audio_chunks(pcm + silence): + await connection.send(audio_message(chunk)) + await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + finally: + finished.set() + + +async def receive_translation( + connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event +) -> str | None: + audio_received = asyncio.Event() + try: + with wave.open(str(output_path), "wb") as output: + output.setnchannels(CHANNELS) + output.setsampwidth(SAMPLE_WIDTH) + output.setframerate(SAMPLE_RATE) + write_stdout("Translation: ", end="", flush=True) + while True: + timeout = ( + OUTPUT_IDLE_TIMEOUT_SECONDS + if sender_finished.is_set() and audio_received.is_set() + else INITIAL_RESPONSE_TIMEOUT_SECONDS + ) + try: + raw_event = await asyncio.wait_for(connection.recv(), timeout=timeout) + except TimeoutError: + if sender_finished.is_set() and audio_received.is_set(): + write_stdout() + return None + return "The translation stream ended without translated audio" + event = json.loads(raw_event) + event_type = event.get("type") + if event_type in AUDIO_EVENT_TYPES: + output.writeframes(base64.b64decode(event.get("delta", ""), validate=True)) + audio_received.set() + elif event_type in TRANSCRIPT_EVENT_TYPES: + write_stdout(event.get("delta", event.get("text", "")), end="", flush=True) + elif event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + except (OSError, wave.Error) as exc: + return f"Unable to write {output_path}: {exc}" + + +async def translate(settings: Settings, pcm: bytes) -> str | None: + url = translation_url(settings.base_url, settings.model) + if not url: + return f"Invalid --base-url: {settings.base_url}" + sender_finished = asyncio.Event() + try: + async with websockets.connect( + url, + additional_headers={"Authorization": f"Bearer {settings.api_key}"}, + proxy=None, + open_timeout=20, + close_timeout=5, + ) as connection: + configuration_error = await configure_session(connection, settings.target_language) + if configuration_error: + return configuration_error + async with asyncio.TaskGroup() as tasks: + receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished)) + tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished)) + return receiver.result() + except Exception as exc: + return f"Translation failed: {type(exc).__name__}: {exc}" + + +def main(argv: Sequence[str] | None = None) -> int: + settings = parse_args(argv) + if isinstance(settings, str): + write_stderr(settings) + return 2 + pcm = read_pcm16_wav(settings.input_wav) + if isinstance(pcm, str): + write_stderr(pcm) + return 2 + error = asyncio.run(translate(settings, pcm)) + if error: + write_stderr(error) + return 1 + write_stdout(f"Translated audio: {settings.output_wav}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/litellm/__init__.py b/litellm/__init__.py index e1da202b9ee..dd5da7bd452 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -29,7 +29,7 @@ def _dev_env_hot_reload_enabled() -> bool: if os.getenv("LITELLM_MODE", "DEV") == "DEV": _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled()) -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from typing import ( Any, Callable, @@ -1477,7 +1477,9 @@ from .realtime_api.main import ( _arealtime, acreate_realtime_client_secret, acreate_realtime_transcription_session, + acreate_realtime_translation_client_secret, arealtime_calls, + arealtime_translation_calls, ) from .responses.main import _aresponses_websocket from .fine_tuning.main import * @@ -1661,6 +1663,9 @@ if TYPE_CHECKING: from .llms.vertex_ai.rerank.transformation import ( VertexAIRerankConfig as VertexAIRerankConfig, ) + from .llms.together_ai.chat.transformation import ( + TogetherAIChatConfig as TogetherAIChatConfig, + ) from .llms.fireworks_ai.rerank.transformation import ( FireworksAIRerankConfig as FireworksAIRerankConfig, ) @@ -1706,9 +1711,6 @@ if TYPE_CHECKING: BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig, ) from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig - from .llms.together_ai.chat.transformation import ( - TogetherAIChatConfig as TogetherAIChatConfig, - ) from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig as VertexGeminiConfig, @@ -1864,6 +1866,9 @@ if TYPE_CHECKING: from .llms.xai.responses.transformation import ( XAIResponsesAPIConfig as XAIResponsesAPIConfig, ) + from .llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig as VertexAIInteractionsConfig, + ) from .llms.litellm_proxy.responses.transformation import ( LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, ) @@ -1891,9 +1896,6 @@ if TYPE_CHECKING: from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) - from .llms.vertex_ai.interactions.transformation import ( - VertexAIInteractionsConfig as VertexAIInteractionsConfig, - ) from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config, diff --git a/litellm/constants.py b/litellm/constants.py index 10c943656f7..9fe8b07b21f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -838,7 +838,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "language", + "languages", + "keywords", "response_format", + "stream", "timestamp_granularities", ] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a279b9f0903..81d913a24b9 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -18,6 +18,7 @@ from litellm.constants import ( DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, ) +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -296,12 +297,29 @@ def _get_additional_costs( return None -def _transcription_usage_has_token_details( +def _transcription_uses_token_pricing( + model: str, + custom_llm_provider: str | None, usage_block: Usage | None, ) -> bool: if usage_block is None: return False + model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + has_token_pricing: Final = any( + model_info.get(field) + for field in ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + ) + ) + if not has_token_pricing and ( + model_info.get("input_cost_per_second") is not None or model_info.get("output_cost_per_second") is not None + ): + return False + prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0 completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0 prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None) @@ -628,7 +646,7 @@ def cost_per_token( data_residency=data_residency, ) elif call_type == "atranscription" or call_type == "transcription": - if _transcription_usage_has_token_details(usage_block): + if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block): return generic_cost_per_token( model=model_without_prefix, usage=usage_block, @@ -1011,6 +1029,23 @@ def get_usage_object( return None +def _get_transcription_usage_duration(completion_response: object) -> float | None: + usage_object: Final = ( + completion_response.get("usage") + if isinstance(completion_response, dict) + else getattr(completion_response, "usage", None) + ) + usage_type: Final = ( + usage_object.get("type") if isinstance(usage_object, dict) else getattr(usage_object, "type", None) + ) + if usage_type != "duration": + return None + seconds: Final = ( + usage_object.get("seconds") if isinstance(usage_object, dict) else getattr(usage_object, "seconds", None) + ) + return normalized_audio_duration_seconds(seconds) + + def _is_known_usage_objects(usage_obj): """Returns True if the usage obj is a known Usage type""" return ( @@ -1600,9 +1635,14 @@ def completion_cost( # the response attribute (for verbose_json responses that # naturally include duration from the provider). _hidden = getattr(completion_response, "_hidden_params", {}) or {} - audio_transcription_file_duration = _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), + provider_duration = _get_transcription_usage_duration(completion_response) + audio_transcription_file_duration = ( + provider_duration + if provider_duration is not None + else _hidden.get( + "audio_transcription_duration", + getattr(completion_response, "duration", 0.0), + ) ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): @@ -2857,6 +2897,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor): _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" +_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed" def _candidate_realtime_token_costs( @@ -2969,7 +3010,22 @@ def handle_realtime_stream_cost_calculation( if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results) else 0.0 ) - total_cost: Final = input_cost_per_token + output_cost_per_token + transcription_cost + translation_cost: Final = handle_realtime_translation_cost_calculation( + results=results, + custom_llm_provider=custom_llm_provider, + litellm_model_name=litellm_model_name, + potential_model_names=potential_model_names, + ) + total_cost: Final = input_cost_per_token + output_cost_per_token + transcription_cost + translation_cost + + additional_costs: Final = { # mutable-ok: logging stores a mutable per-request cost breakdown + key: value + for key, value in ( + ("transcription_cost", transcription_cost), + ("translation_cost", translation_cost), + ) + if value > 0 + } _store_cost_breakdown_in_logging_obj( litellm_logging_obj=litellm_logging_obj, @@ -2977,13 +3033,61 @@ def handle_realtime_stream_cost_calculation( completion_tokens_cost_usd_dollar=output_cost_per_token, cost_for_built_in_tools_cost_usd_dollar=0.0, total_cost_usd_dollar=total_cost, - additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None, data_residency=data_residency, + additional_costs=additional_costs or None, ) return total_cost +def handle_realtime_translation_cost_calculation( + results: OpenAIRealtimeStreamList, + custom_llm_provider: str, + litellm_model_name: str, + potential_model_names: Sequence[str | None] = (), +) -> float: + usage_events: Final = tuple( + usage + for result in results + if result.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE + if isinstance(usage := result.get("usage"), dict) + ) + input_seconds: Final = sum( + seconds + for usage in usage_events + if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None + ) + output_seconds: Final = sum( + seconds + for usage in usage_events + if (seconds := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None + ) + if input_seconds <= 0 and output_seconds <= 0: + return 0.0 + model_infos: Final = tuple( + _get_model_info_or_none(model, custom_llm_provider) + for model in (*potential_model_names, litellm_model_name) + if model is not None + ) + input_cost_per_second: Final = next( + ( + rate + for info in model_infos + if (rate := _declared_transcription_rate(info, ("input_cost_per_second",))) is not None + ), + 0.0, + ) + output_cost_per_second: Final = next( + ( + rate + for info in model_infos + if (rate := _declared_transcription_rate(info, ("output_cost_per_second",))) is not None + ), + 0.0, + ) + return input_seconds * input_cost_per_second + output_seconds * output_cost_per_second + + def handle_realtime_transcription_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py new file mode 100644 index 00000000000..979a1847362 --- /dev/null +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import datetime +import traceback +from collections.abc import AsyncIterator, Iterator +from typing import Final, Protocol + +from openai import AsyncStream, Stream +from openai.types.audio import ( + TranscriptionStreamEvent, + TranscriptionTextDeltaEvent, + TranscriptionTextDoneEvent, +) + +from litellm.types.utils import TranscriptionResponse + + +class TranscriptionStreamLogging(Protocol): + def success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def handle_sync_success_callbacks_for_async_calls( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + +class _TranscriptionEventCollector: + def __init__(self, duration: float | None) -> None: + self.duration = duration + self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event + self.done_event: TranscriptionTextDoneEvent | None = None + + def add(self, event: TranscriptionStreamEvent) -> None: + if isinstance(event, TranscriptionTextDeltaEvent): + self.text_deltas.append(event.delta) + elif isinstance(event, TranscriptionTextDoneEvent): + self.done_event = event + + def response(self) -> TranscriptionResponse: + done_event: Final = self.done_event + done_languages: Final = getattr(done_event, "languages", None) if done_event is not None else None + response: Final = TranscriptionResponse( + text=done_event.text if done_event is not None else "".join(self.text_deltas), + usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, + languages=( + [ # mutable-ok: the response model requires a concrete serialized language list + language.model_dump() for language in done_languages + ] + if done_languages is not None + else None + ), + ) + if self.duration is not None: + response.set_audio_transcription_duration(self.duration) + return response + + +class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): + def __init__( + self, + stream: Stream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + ) -> None: + self.__dict__.update(stream.__dict__) + self._logging_obj = logging_obj + self._start_time = start_time + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + def _logging_iterator( + self, source_iterator: Iterator[TranscriptionStreamEvent] + ) -> Iterator[TranscriptionStreamEvent]: + try: + for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(exception, traceback.format_exc(), self._start_time, end_time) + raise + finally: + self._finalize() + + def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + if self._collector.done_event is None and self._collector.duration is None: + return + self._logging_obj.success_handler( + self._collector.response(), + self._start_time, + datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract + ) + + def close(self) -> None: + try: + super().close() + finally: + self._finalize() + + +class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): + def __init__( + self, + stream: AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + ) -> None: + self.__dict__.update(stream.__dict__) + self._logging_obj = logging_obj + self._start_time = start_time + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + async def _logging_iterator( + self, source_iterator: AsyncIterator[TranscriptionStreamEvent] + ) -> AsyncIterator[TranscriptionStreamEvent]: + try: + async for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(exception, traceback.format_exc(), self._start_time, end_time) + await self._logging_obj.async_failure_handler( + exception, + traceback.format_exc(), + self._start_time, + end_time, + ) + raise + finally: + await self._finalize() + + async def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + if self._collector.done_event is None and self._collector.duration is None: + return + response: Final = self._collector.response() + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.handle_sync_success_callbacks_for_async_calls( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + await self._logging_obj.async_success_handler( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + + async def close(self) -> None: + try: + await super().close() + finally: + await self._finalize() + + +def wrap_transcription_stream( + stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, +) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream: + if isinstance(stream, AsyncStream): + return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time) + return LoggingTranscriptionStream(stream, logging_obj, start_time) diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index dab3e48f91a..9fdfce057d9 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -3,6 +3,7 @@ Utils used for litellm.transcription() and litellm.atranscription() """ import hashlib +import math import os from dataclasses import dataclass from typing import Final @@ -17,6 +18,16 @@ from litellm.types.files import ( from litellm.types.utils import FileTypes +def normalized_audio_duration_seconds(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + seconds: Final = float(value) + except OverflowError: + return None + return seconds if math.isfinite(seconds) and seconds >= 0 else None + + @dataclass class ProcessedAudioFile: """ diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 5af669591f7..c6d96f83e7c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3287,9 +3287,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE if "async_complete_streaming_response" in self.model_call_details: return # break out of this. - complete_streaming_response: Final[ - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None - ] = self._get_assembled_streaming_response( + complete_streaming_response: Final = self._get_assembled_streaming_response( result=result, start_time=start_time, end_time=end_time, @@ -4089,10 +4087,17 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: list[object], - ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None: + ) -> ( + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None + ): if self.stream is not True: return None - if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse): + if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)): return result elif isinstance( result, diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index a02c40b7611..a34e4de2dca 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -31,9 +31,13 @@ class TranscriptionUsageObjectTransformation: prompt_tokens=usage_object.input_tokens, completion_tokens=usage_object.output_tokens, total_tokens=usage_object.total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=usage_object.input_token_details.text_tokens, - audio_tokens=usage_object.input_token_details.audio_tokens, + prompt_tokens_details=( + PromptTokensDetailsWrapper( + text_tokens=usage_object.input_token_details.text_tokens, + audio_tokens=usage_object.input_token_details.audio_tokens, + ) + if usage_object.input_token_details is not None + else None ), ) return None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 795911cafe2..a63f09c7bef 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -911,6 +911,11 @@ def calculate_cache_writing_cost( class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int cache_hit_audio_tokens: ReadOnly[int] + + cached_text_tokens: ReadOnly[int] + cached_audio_tokens: ReadOnly[int] + cached_image_tokens: ReadOnly[int] + has_cached_tokens_details: ReadOnly[bool] cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None text_tokens: int @@ -997,6 +1002,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, cache_hit_audio_tokens=cached_audio_tokens, + cached_text_tokens=cached_text_tokens, + cached_audio_tokens=cached_audio_tokens, + cached_image_tokens=cached_image_tokens, + has_cached_tokens_details=cached_tokens_details is not None, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, @@ -1080,15 +1089,11 @@ def _calculate_input_cost( prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing - cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] - audio_cache_read_rate: Final = _get_cost_per_unit( - model_info, - _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost - prompt_cost += float(cache_hit_audio_tokens) * ( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost + prompt_cost += _calculate_cache_read_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=model_info, + cache_read_cost=cache_read_cost, + service_tier=service_tier, ) ### AUDIO COST @@ -1168,6 +1173,38 @@ def _calculate_input_cost( return prompt_cost +def _calculate_cache_read_cost( + prompt_tokens_details: PromptTokensDetailsResult, + model_info: ModelInfo, + cache_read_cost: float, + service_tier: str | None, +) -> float: + cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"] + cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"] + cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"] + classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens + unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0) + total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost + float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost + ) + + if cached_audio_tokens: + cached_audio_cost_key: Final = _get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier) + cached_audio_cost: Final = _get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached audio contributes to cache-read cost + float(cached_audio_tokens) * float(cached_audio_cost or 0.0) + ) + + if cached_image_tokens: + cached_image_cost_key: Final = _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier) + cached_image_cost: Final = _get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached images contribute to cache-read cost + float(cached_image_tokens) * float(cached_image_cost or 0.0) + ) + + return total_cost + + def _get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: """ Resolve the per-model regional-processing uplift multiplier for a given @@ -1329,6 +1366,10 @@ def generic_cost_per_token( prompt_tokens_details = PromptTokensDetailsResult( cache_hit_tokens=0, cache_hit_audio_tokens=0, + cached_text_tokens=0, + cached_audio_tokens=0, + cached_image_tokens=0, + has_cached_tokens_details=False, cache_creation_tokens=0, cache_creation_token_details=None, text_tokens=usage.prompt_tokens, @@ -1503,6 +1544,7 @@ class BilledTokenRates: cache_creation_input_token_cost: float cache_creation_input_token_cost_above_1hr: float output_cost_per_reasoning_token: float + cache_read_input_image_token_cost: float | None = None def scaled(self, multiplier: float) -> "BilledTokenRates": if multiplier == 1.0: @@ -1515,6 +1557,11 @@ class BilledTokenRates: cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier, output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, + cache_read_input_image_token_cost=( + self.cache_read_input_image_token_cost * multiplier + if self.cache_read_input_image_token_cost is not None + else None + ), ) @@ -1618,6 +1665,11 @@ def _cost_map_billed_rates( cache_creation_input_token_cost=cache_creation_cost_rate, cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate, output_cost_per_reasoning_token=reasoning_rate, + cache_read_input_image_token_cost=_get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier), + None, + ), ).scaled(multiplier) @@ -1693,6 +1745,12 @@ def get_token_type_cost_breakdown( cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( usage ) + cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"] + image_cache_read_rate: Final = ( + rates.cache_read_input_image_token_cost + if rates.cache_read_input_image_token_cost is not None + else rates.cache_read_input_token_cost + ) cache_creation_cost: Final = ( float(cache_creation_tokens) * rates.cache_creation_input_token_cost if custom_cost_per_token is not None @@ -1706,8 +1764,9 @@ def get_token_type_cost_breakdown( return TokenTypeCostBreakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, cache_read_cost=( - float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost + float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + + float(cached_image_tokens) * image_cache_read_rate ), cache_creation_cost=cache_creation_cost, rates=rates, diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 9ea730a873f..ce64b806112 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -38,6 +38,7 @@ from litellm.types.utils import ( StreamingChoices, TextChoices, TextCompletionResponse, + TranscriptionDetectedLanguage, TranscriptionResponse, TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, @@ -770,9 +771,11 @@ def convert_to_model_response_object( model_response_object.data = response_object["data"] if "usage" in response_object and response_object["usage"] is not None: - model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) - model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) - model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) + embedding_usage: Final = model_response_object.usage or Usage() + embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) + embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) + embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0) + model_response_object.usage = embedding_usage if start_time is not None and end_time is not None: model_response_object._response_ms = ( @@ -815,6 +818,12 @@ def convert_to_model_response_object( if key in response_object: setattr(model_response_object, key, response_object[key]) + if "languages" in response_object and response_object["languages"] is not None: + transcription_response: Final = model_response_object + transcription_response.languages = tuple( + TranscriptionDetectedLanguage.model_validate(language) for language in response_object["languages"] + ) + if "usage" in response_object and response_object["usage"] is not None: tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index d2fbb26bb02..4d2d4dde999 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,4 +1,5 @@ import asyncio +import base64 import json import traceback from collections.abc import Coroutine, Mapping, Sequence @@ -11,6 +12,7 @@ from typing_extensions import ReadOnly import litellm from litellm._logging import redact_internal_details_from_client_message, verbose_logger from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( @@ -19,6 +21,8 @@ from litellm.types.llms.openai import ( OpenAIRealtimeResponseDelta, OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamSessionEvents, + OpenAIRealtimeTranslationClosedEvent, + OpenAIRealtimeTranslationDurationUsage, ) from litellm.types.realtime import ALL_DELTA_TYPES @@ -137,6 +141,7 @@ class RealTimeStreaming: force_transcription_model: str | None = None, event_normalizer: RealtimeEventNormalizer | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + translation_session: bool = False, ): self.websocket: _ClientWebSocket = websocket self.backend_ws = backend_ws @@ -148,6 +153,11 @@ class RealTimeStreaming: self.input_messages: list[dict[str, str]] = [] self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] + self._is_translation_session = translation_session + self._translation_input_seconds = 0.0 + self._translation_output_audio_bytes = 0 + self._translation_output_bytes_per_second = 48000.0 + self._translation_usage_finalized = False # Detect whether the client is explicitly opting into the beta protocol. self._client_wants_beta = self._detect_beta_header(websocket) @@ -196,6 +206,7 @@ class RealTimeStreaming: # their input_audio_transcription.completed usage drives duration-based cost. self._force_transcription_model = force_transcription_model self._is_transcription_session: bool = force_transcription_model is not None + self._bound_nested_transcription_model: str | None = None # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer). self._event_normalizer = event_normalizer @@ -410,6 +421,7 @@ class RealTimeStreaming: async def log_messages(self): """Log messages in list""" + self._finalize_translation_usage() if self.logging_obj: if self.input_messages: self.logging_obj.model_call_details["messages"] = self.input_messages @@ -424,6 +436,140 @@ class RealTimeStreaming: ) self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True + def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None: + if not self._is_translation_session: + return + if event_obj.get("type") == "session.closed": + usage: Final = event_obj.get("usage") + output_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("output_seconds")) if isinstance(usage, dict) else None + ) + reported_input_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("input_seconds")) if isinstance(usage, dict) else None + ) + input_seconds: Final = ( + reported_input_seconds + if reported_input_seconds is not None + else self._translation_input_seconds or None + ) + synthetic_output_seconds: Final = ( + self._translation_output_audio_bytes / self._translation_output_bytes_per_second + if self._translation_output_audio_bytes > 0 + else None + ) + resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds + if input_seconds is not None or resolved_output_seconds is not None: + if self._should_store_message(event_obj): + supplemental_usage: Final = OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=float(input_seconds or 0.0) if reported_input_seconds is None else 0.0, + output_seconds=float(synthetic_output_seconds or 0.0) if output_seconds is None else 0.0, + ) + if ( + supplemental_usage.get("input_seconds", 0.0) > 0 + or supplemental_usage.get("output_seconds", 0.0) > 0 + ): + self.messages.append( + OpenAIRealtimeTranslationClosedEvent( + type="session.closed", + usage=supplemental_usage, + ) + ) + else: + normalized_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=input_seconds, + output_seconds=float(resolved_output_seconds or 0.0), + ) + if input_seconds is not None + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=float(resolved_output_seconds or 0.0) + ) + ) + self.messages.append( + OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=normalized_usage) + ) + self._translation_usage_finalized = True + return + self._capture_translation_output_format(event_obj) + if event_obj.get("type") not in ( + "session.output_audio.delta", + "response.output_audio.delta", + "response.audio.delta", + ): + return + delta: Final = event_obj.get("delta") + if not isinstance(delta, str): + return + try: + decoded: Final = base64.b64decode(delta, validate=True) + except (ValueError, TypeError): + return + self._translation_output_audio_bytes += len(decoded) + + def _capture_translation_input_audio(self, message: str) -> None: + if not self._is_translation_session: + return + try: + event: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return + if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str): + return + try: + decoded: Final = base64.b64decode(audio, validate=True) + except (ValueError, TypeError): + return + self._translation_input_seconds += len(decoded) / 48000.0 + + def _capture_translation_output_format(self, event_obj: Mapping[str, object]) -> None: + session: Final = event_obj.get("session") + if not isinstance(session, dict): + return + audio: Final = session.get("audio") + output: Final = audio.get("output") if isinstance(audio, dict) else None + audio_format: Final = output.get("format") if isinstance(output, dict) else None + if isinstance(audio_format, str): + if audio_format in ("g711_ulaw", "g711_alaw"): + self._translation_output_bytes_per_second = 8000.0 + return + if not isinstance(audio_format, dict): + return + format_type: Final = audio_format.get("type") + rate: Final = audio_format.get("rate") + if not isinstance(rate, (int, float)) or rate <= 0: + return + if format_type == "audio/pcm": + self._translation_output_bytes_per_second = float(rate) * 2 + elif format_type in ("audio/pcmu", "audio/pcma"): + self._translation_output_bytes_per_second = float(rate) + + def _finalize_translation_usage(self) -> None: + if self._translation_usage_finalized: + return + for event in self.messages: + if event.get("type") != "session.closed": + continue + if ( + isinstance(event_usage := event.get("usage"), dict) + and normalized_audio_duration_seconds(event_usage.get("output_seconds")) is not None + ): + self._translation_usage_finalized = True + return + if self._translation_output_audio_bytes == 0 and self._translation_input_seconds == 0: + return + output_seconds: Final = self._translation_output_audio_bytes / self._translation_output_bytes_per_second + synthetic_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", input_seconds=self._translation_input_seconds, output_seconds=output_seconds + ) + if self._translation_input_seconds > 0 + else OpenAIRealtimeTranslationDurationUsage(type="duration", output_seconds=output_seconds) + ) + self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) + self._translation_usage_finalized = True + async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. @@ -436,7 +582,7 @@ class RealTimeStreaming: backend, False if the provider transformation produced no output and the message was effectively dropped. """ - message = self._enforce_transcription_session_model(message) + message = await self._apply_nested_transcription_model_policy(message) if self.provider_config: transformed: Final = self.provider_config.transform_realtime_request( message, self.model, self.session_configuration_request @@ -474,10 +620,97 @@ class RealTimeStreaming: if is_content_message: self._content_sent_after_setup = True sent = True + if sent: + self._capture_translation_input_audio(message) return sent await self.backend_ws.send(message) + self._capture_translation_input_audio(message) return True + async def _apply_nested_transcription_model_policy(self, message: str) -> str: + if self._force_transcription_model is not None: + return self._enforce_transcription_session_model(message) + if self._is_translation_session: + return await self._enforce_translation_nested_transcription_model(message) + return message + + def _session_update_message_obj(self, message: str) -> Mapping[str, object] | None: + try: + message_obj: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return None + if message_obj.get("type") not in ( + "session.update", + "transcription_session.update", + ): + return None + return message_obj + + def _nested_transcription_models_from_session( + self, + session: Mapping[str, object], + ) -> tuple[str, ...]: + audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None + nested_model: Final = self._transcription_model_value(nested_transcription) + flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription")) + return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None)) + + def _transcription_model_value(self, transcription_config: object) -> str | None: + if not isinstance(transcription_config, dict): + return None + model: Final = transcription_config.get("model") + if isinstance(model, str) and model: + return model + return None + + def _rewrite_session_update_transcription_model(self, message: str, authorized_model: str) -> str: + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: + return message + session: Final = message_obj.get("session") + if not isinstance(session, dict): + return message + + transcription: Final = session.get("input_audio_transcription") + rewrite_flat: Final = isinstance(transcription, dict) and transcription.get("model") != authorized_model + if isinstance(transcription, dict) and rewrite_flat: + session["input_audio_transcription"] = { + **transcription, + "model": authorized_model, + } + + audio: Final = session.get("audio") + audio_input: Final = audio.get("input") if isinstance(audio, dict) else None + nested_transcription: Final = audio_input.get("transcription") if isinstance(audio_input, dict) else None + rewrite_nested: Final = ( + isinstance(audio, dict) + and isinstance(audio_input, dict) + and isinstance(nested_transcription, dict) + and nested_transcription.get("model") != authorized_model + ) + if ( + isinstance(audio, dict) + and isinstance(audio_input, dict) + and isinstance(nested_transcription, dict) + and rewrite_nested + ): + session["audio"] = { + **audio, + "input": { + **audio_input, + "transcription": { + **nested_transcription, + "model": authorized_model, + }, + }, + } + + if not rewrite_flat and not rewrite_nested: + return message + return json.dumps(message_obj) + def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -495,56 +728,49 @@ class RealTimeStreaming: if self._force_transcription_model is None: return message - try: - message_obj: Final = _decode_json_object(message) - except (json.JSONDecodeError, TypeError): + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message + session: Final = message_obj.get("session") + if isinstance(session, dict) and session.get("type") == "transcription": + self._is_transcription_session = True + return self._rewrite_session_update_transcription_model(message, self._force_transcription_model) - if message_obj.get("type") not in ( - "session.update", - "transcription_session.update", - ): + async def _enforce_translation_nested_transcription_model(self, message: str) -> str: + if self._bound_nested_transcription_model is not None: + return self._rewrite_session_update_transcription_model(message, self._bound_nested_transcription_model) + + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message - session: Final = message_obj.get("session") if not isinstance(session, dict): return message - - if session.get("type") == "transcription": - self._is_transcription_session = True - - authorized_model: Final = self._force_transcription_model - changed = False - - transcription: Final = session.get("input_audio_transcription") - if isinstance(transcription, dict) and transcription.get("model") != authorized_model: - session["input_audio_transcription"] = { - **transcription, - "model": authorized_model, - } - changed = True - - audio: Final = session.get("audio") - if isinstance(audio, dict): - audio_input: Final = audio.get("input") - if isinstance(audio_input, dict): - nested_transcription: Final = audio_input.get("transcription") - if isinstance(nested_transcription, dict) and nested_transcription.get("model") != authorized_model: - session["audio"] = { - **audio, - "input": { - **audio_input, - "transcription": { - **nested_transcription, - "model": authorized_model, - }, - }, - } - changed = True - - if not changed: + nested_models: Final = self._nested_transcription_models_from_session(session) + if not nested_models: return message - return json.dumps(message_obj) + + valid_token: Final = self.user_api_key_dict + if valid_token is None: + return message + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import can_key_call_resolved_model + from litellm.proxy.proxy_server import llm_model_list, llm_router + + if not isinstance(valid_token, UserAPIKeyAuth): + return message + + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + bound_model: Final = nested_models[0] + self._bound_nested_transcription_model = bound_model + return self._rewrite_session_update_transcription_model(message, bound_model) def _uses_deferred_backend_setup(self) -> bool: """True when setup is deferred until the client's first session.update.""" @@ -942,7 +1168,10 @@ class RealTimeStreaming: async def _handle_provider_config_message(self, raw_response: str) -> None: """Process a backend message when a provider_config is set (transformed path).""" - returned_object: Final = self.provider_config.transform_realtime_response( + provider_config: Final = self.provider_config + if provider_config is None: + raise RuntimeError("Provider response handling requires a provider configuration") + returned_object: Final = provider_config.transform_realtime_response( raw_response, self.model, self.logging_obj, @@ -969,6 +1198,8 @@ class RealTimeStreaming: for event in events: if self._should_drop_event_from_client(event): continue + if isinstance(event, dict): + self._capture_translation_output_audio(event) is_session_created_event = isinstance(event, dict) and event.get("type") == "session.created" if is_session_created_event: if self._uses_deferred_backend_setup() and not self._backend_setup_complete: @@ -1103,6 +1334,7 @@ class RealTimeStreaming: if self._should_drop_event_from_client(event): continue + self._capture_translation_output_audio(event) if await self._handle_raw_backend_message(event, raw_response): continue @@ -1507,6 +1739,8 @@ class RealTimeStreaming: session = client_event.get("session", {}) if isinstance(session, dict): session = self._remap_beta_session_to_ga(session) + if self._is_translation_session: + session.pop("type", None) msg_obj["session"] = session message = json.dumps(msg_obj) diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index 564ec94ba6b..e5675d4c82f 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,11 +1,13 @@ from collections.abc import Coroutine from typing import TYPE_CHECKING, Any, Final -from openai import AsyncAzureOpenAI, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from pydantic import BaseModel +import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name +from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data from litellm.types.utils import FileTypes from litellm.utils import ( TranscriptionResponse, @@ -40,15 +42,27 @@ class AzureAudioTranscription(AzureChatCompletion): custom_llm_provider: str = "azure", ) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]: data: Final = {"model": model, "file": audio_file, **optional_params} + sdk_data: Final = sdk_compatible_transcription_request_data(data) + model_info: Final = litellm.model_cost.get(f"azure/{model}") + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + requires_deployment_api: Final = model_info is None or ( + provider_specific_entry is not None and provider_specific_entry.get("transcription_deployment_api") == 1 + ) + resolved_api_version: Final = ( + litellm.AZURE_DEFAULT_API_VERSION + if requires_deployment_api and api_version in ("v1", "latest", "preview") + else api_version + ) if atranscription is True: return self.async_audio_transcriptions( audio_file=audio_file, - data=data, + data=sdk_data, model_response=model_response, timeout=timeout, api_key=api_key, api_base=api_base, + api_version=resolved_api_version, client=client, max_retries=max_retries, logging_obj=logging_obj, @@ -58,7 +72,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) azure_client: Final = self.get_azure_openai_client( - api_version=api_version, + api_version=resolved_api_version, api_base=api_base, api_key=api_key, model=model, @@ -66,7 +80,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(azure_client, AzureOpenAI): + if not isinstance(azure_client, (AzureOpenAI, OpenAI)): raise AzureOpenAIError( status_code=500, message="azure_client is not an instance of AzureOpenAI", @@ -85,10 +99,13 @@ class AzureAudioTranscription(AzureChatCompletion): ) response: Final = azure_client.audio.transcriptions.create( - **data, + **sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options timeout=timeout, ) + if data.get("stream") is True: + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -137,7 +154,7 @@ class AzureAudioTranscription(AzureChatCompletion): client=client, litellm_params=litellm_params, ) - if not isinstance(async_azure_client, AsyncAzureOpenAI): + if not isinstance(async_azure_client, (AsyncAzureOpenAI, AsyncOpenAI)): raise AzureOpenAIError( status_code=500, message="async_azure_client is not an instance of AsyncAzureOpenAI", @@ -155,8 +172,15 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) + if data.get("stream") is True: + return await async_azure_client.audio.transcriptions.create( + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, + ) + raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( - **data, timeout=timeout + **data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options + timeout=timeout, ) headers: Final = dict(raw_response.headers) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index df74975ad0f..7b38fc85365 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -8,6 +8,7 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, Protocol, cast +import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams @@ -39,16 +40,36 @@ async def forward_messages(client_ws: Any, backend_ws: Any): pass +def azure_realtime_requires_ga(model: str) -> bool: + try: + azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + try: + openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + return False + openai_entry: Final = openai_model_info.get("provider_specific_entry") + return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1 + azure_entry: Final = azure_model_info.get("provider_specific_entry") + return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1 + + def azure_realtime_protocol_for_client( configured_protocol: object, *, + model: str, + realtime_mode: str, query_params: RealtimeQueryParams | None, websocket: ScopedWebSocket, ) -> str: + if azure_realtime_requires_ga(model): + if isinstance(configured_protocol, str) and configured_protocol.upper() not in ("GA", "V1"): + raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API") + return "GA" + if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription": + return "GA" if isinstance(configured_protocol, str) and configured_protocol: return configured_protocol - if (query_params or {}).get("intent") == "transcription": - return "GA" return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" @@ -83,6 +104,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version: str | None, realtime_protocol: str | None = None, query_params: RealtimeQueryParams | None = None, + realtime_mode: str = "realtime", ) -> str: """ Construct Azure realtime WebSocket URL. @@ -114,18 +136,26 @@ class AzureOpenAIRealtime(AzureChatCompletion): ) intent: Final = (query_params or {}).get("intent") - if _is_ga: - path = "/openai/v1/realtime" - query_parts = [] - if intent != "transcription" and (query_params is None or "model" in query_params): - query_parts.append(urlencode({"model": model})) - else: - # Default to beta path for backwards compatibility - path = "/openai/realtime" - query_parts = [urlencode({"api-version": api_version, "deployment": model})] + path: Final = ( + "/openai/v1/realtime/translations" + if realtime_mode == "translation" + else "/openai/v1/realtime" + if _is_ga + else "/openai/realtime" + ) + base_query_parts: Final = ( + (urlencode((("model", model),)),) + if realtime_mode == "translation" + else ( + (urlencode((("model", model),)),) + if intent != "transcription" and (query_params is None or "model" in query_params) + else () + ) + if _is_ga + else (urlencode((("api-version", api_version), ("deployment", model))),) + ) - if intent: - query_parts.append(urlencode({"intent": intent})) + query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts qs: Final = "&".join(query_parts) return f"{api_base}{path}?{qs}" if qs else f"{api_base}{path}" @@ -145,6 +175,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", ): import websockets from websockets.asyncio.client import ClientConnection @@ -161,6 +192,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, ) auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token) @@ -184,6 +216,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): force_transcription_model=( model if (query_params or {}).get("intent") == "transcription" else None ), + translation_session=realtime_mode == "translation", ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 633c2b3b6d0..9b37d1faa95 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -1,21 +1,45 @@ """Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" +from collections.abc import Mapping from typing import Final import litellm +from litellm.llms.azure.common_utils import get_azure_ad_token from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +from .handler import azure_realtime_requires_ga class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): + @staticmethod + def _uses_ga_api(model: str, api_version: str | None) -> bool: + return api_version in ("preview", "latest", "v1") or azure_realtime_requires_ga(model) + def get_api_base(self, api_base: str | None, **kwargs) -> str: return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or "" def get_api_key(self, api_key: str | None, **kwargs) -> str: return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or "" + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + if api_key: + return extra_headers + azure_ad_token: Final = get_azure_ad_token(litellm_params) + if azure_ad_token is None: + return extra_headers + return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"} + def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") + if self._uses_ga_api(model, api_version): + return f"{base}/openai/v1/realtime/client_secrets" version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{base}/openai/realtime/client_secrets?api-version={version}" @@ -25,22 +49,38 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): model: str, api_key: str | None = None, ) -> dict: - return { + validated_headers: Final = { # mutable-ok: provider authentication headers are extended before dispatch **headers, - "api-key": api_key or "", "Content-Type": "application/json", } + if api_key: + validated_headers["api-key"] = api_key + return validated_headers def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") + if self._uses_ga_api(model, api_version): + return f"{base}/openai/v1/realtime/calls" version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{base}/openai/realtime/calls?api-version={version}" def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base: Final = self.get_api_base(api_base).rstrip("/") + if self._uses_ga_api(model, api_version): + return f"{base}/openai/v1/realtime/transcription_sessions" version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{base}/openai/realtime/transcription_sessions?api-version={version}" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base: Final = self.get_api_base(api_base).rstrip("/") + return f"{base}/openai/v1/realtime/translations/client_secrets" + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base: Final = self.get_api_base(api_base).rstrip("/") + return f"{base}/openai/v1/realtime/translations/calls" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: return { "api-key": ephemeral_key, diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..2078b063453 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -11,6 +11,23 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse + +def sdk_compatible_transcription_request_data(data: dict) -> dict: + extension_keys: Final = ("keywords", "languages") + extension_body: Final = {key: data[key] for key in extension_keys if key in data} + if not extension_body: + return data + + existing_extra_body: Final = data.get("extra_body") + return { + **{key: value for key, value in data.items() if key not in extension_keys}, + "extra_body": { + **(existing_extra_body if isinstance(existing_extra_body, dict) else {}), + **extension_body, + }, + } + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index 43a80edb493..de5032be381 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow: """ from abc import ABC, abstractmethod +from collections.abc import Mapping from typing import Final import httpx +from litellm.types.router import GenericLiteLLMParams + class BaseRealtimeHTTPConfig(ABC): """ @@ -49,6 +52,14 @@ class BaseRealtimeHTTPConfig(ABC): explicit api_key → litellm.api_key → env var → "" """ + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + return extra_headers + # ------------------------------------------------------------------ # # client_secrets endpoint # # ------------------------------------------------------------------ # @@ -63,6 +74,12 @@ class BaseRealtimeHTTPConfig(ABC): base = base.removesuffix("/v1") return f"{base}/v1/realtime/transcription_sessions" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/client_secrets" + @abstractmethod def validate_environment( self, @@ -86,6 +103,10 @@ class BaseRealtimeHTTPConfig(ABC): base: Final = (api_base or "").rstrip("/") return f"{base}/v1/realtime/calls" + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/calls" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: """ Build headers for the realtime_calls POST. diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 60d12337447..ae331a21f7b 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -24,7 +24,9 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse import httpx from httpx import USE_CLIENT_DEFAULT from httpx._types import FileContent +from openai import AsyncOpenAI from openai.types.file_deleted import FileDeleted +from openai.types.realtime import RealtimeSessionCreateRequestParam from typing_extensions import ReadOnly import litellm @@ -6174,9 +6176,10 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, + use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/client_secrets to upstream provider. @@ -6184,6 +6187,33 @@ class BaseLLMHTTPHandler: Uses provider_config (BaseRealtimeHTTPConfig) for URL construction and header auth when available; falls back to the legacy OpenAI-style defaults. """ + if use_openai_sdk: + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) + logging_obj.pre_call( + input=request_data, + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "complete_input_dict": request_data, + "api_base": normalized_api_base, + }, + ) + try: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + key: str(value) + for key, value in (extra_headers or {}).items() # mutable-ok: SDK requires concrete headers + }, + ) + raw_response: Final = await configured_client.post( + "/realtime/client_secrets", + cast_to=httpx.Response, + body=request_data, + ) + return self._decoded_realtime_sdk_response(raw_response) + finally: + if owns_client: + await openai_client.close() return await self._async_realtime_session_post( endpoint="client_secrets", api_base=api_base, @@ -6207,8 +6237,8 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, ) -> httpx.Response: """Forward POST /v1/realtime/transcription_sessions to upstream provider.""" @@ -6226,9 +6256,8 @@ class BaseLLMHTTPHandler: api_version=api_version, ) - async def _async_realtime_session_post( + async def async_realtime_translation_client_secret_handler( self, - endpoint: Literal["client_secrets", "transcription_sessions"], api_base: str, api_key: str, request_data: dict[str, object], @@ -6236,8 +6265,63 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, + api_version: str | None = None, + use_openai_sdk: bool = False, + ) -> httpx.Response: + if use_openai_sdk: + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client(api_base, api_key, client) + logging_obj.pre_call( + input=request_data, + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "complete_input_dict": request_data, + "api_base": normalized_api_base, + }, + ) + try: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers=MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ), + ) + raw_response: Final = await configured_client.post( + "/realtime/translations/client_secrets", + cast_to=httpx.Response, + body=request_data, + ) + return self._decoded_realtime_sdk_response(raw_response) + finally: + if owns_client: + await openai_client.close() + return await self._async_realtime_session_post( + endpoint="translation_client_secrets", + api_base=api_base, + api_key=api_key, + request_data=request_data, + logging_obj=logging_obj, + timeout=timeout, + provider_config=provider_config, + model=model, + extra_headers=extra_headers, + client=client, + api_version=api_version, + ) + + async def _async_realtime_session_post( + self, + endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"], + api_base: str, + api_key: str, + request_data: dict[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + provider_config: BaseRealtimeHTTPConfig | None = None, + model: str | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, ) -> httpx.Response: """ @@ -6259,13 +6343,20 @@ class BaseLLMHTTPHandler: url = provider_config.get_transcription_session_url( api_base=api_base, model=model or "", api_version=api_version ) + elif endpoint == "translation_client_secrets": + url = provider_config.get_translation_client_secret_url( + api_base=api_base, model=model or "", api_version=api_version + ) else: url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version) headers: dict[str, object] = provider_config.validate_environment( headers={}, model=model or "", api_key=api_key ) else: - url = f"{api_base.rstrip('/')}/v1/realtime/{endpoint}" + endpoint_path: Final = ( + "translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint + ) + url = f"{api_base.rstrip('/')}/v1/realtime/{endpoint_path}" headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", @@ -6299,6 +6390,92 @@ class BaseLLMHTTPHandler: ) raise + async def _async_realtime_calls_sdk( + self, + api_base: str, + openai_ephemeral_key: str, + sdp_text: str, + session_data: Mapping[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + extra_headers: Mapping[str, object] | None, + client: object | None, + translation: bool, + ) -> httpx.Response: + openai_client, owns_client, normalized_api_base = self._realtime_sdk_client( + api_base, openai_ephemeral_key, client + ) + logging_obj.pre_call( + input="realtime_sdp_offer", + api_key="", + additional_args={ # mutable-ok: logging owns a mutable request metadata payload + "api_base": normalized_api_base, + "session": session_data, + }, + ) + try: + if translation: + configured_client: Final = openai_client.with_options( + timeout=timeout, + set_default_headers={ # mutable-ok: OpenAI SDK accepts a mutable custom-header mapping + "Content-Type": "application/sdp", + **MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ), + }, + ) + translation_response: Final = await configured_client.post( + "/realtime/translations/calls", + cast_to=httpx.Response, + content=sdp_text.encode("utf-8"), + ) + return self._decoded_realtime_sdk_response(translation_response) + realtime_session_data: Final = cast( # cast-ok: endpoint validation produced an OpenAI realtime session + RealtimeSessionCreateRequestParam, + session_data, + ) + sdk_extra_headers: Final = MappingProxyType( + {key: str(value) for key, value in (extra_headers or MappingProxyType({})).items()} + ) + raw_response: Final = await openai_client.realtime.calls.with_raw_response.create( + sdp=sdp_text, + session=realtime_session_data, + extra_headers=sdk_extra_headers, + timeout=timeout, + ) + return self._decoded_realtime_sdk_response(raw_response.http_response) + finally: + if owns_client: + await openai_client.close() + + @staticmethod + def _realtime_sdk_client(api_base: str, api_key: str, client: object | None) -> tuple[AsyncOpenAI, bool, str]: + trimmed_api_base: Final = api_base.rstrip("/") + normalized_api_base: Final = trimmed_api_base if trimmed_api_base.endswith("/v1") else f"{trimmed_api_base}/v1" + if isinstance(client, AsyncOpenAI): + return client, False, normalized_api_base + return AsyncOpenAI(api_key=api_key, base_url=normalized_api_base, max_retries=0), True, normalized_api_base + + @staticmethod + def _decoded_realtime_sdk_response(response: httpx.Response) -> httpx.Response: + headers: Final = { # mutable-ok: httpx accepts a concrete response header mapping + key: value + for key, value in response.headers.items() + if key.lower() not in ("content-encoding", "content-length", "transfer-encoding") + } + return httpx.Response( + status_code=response.status_code, + headers=headers, + content=response.content, + request=response.request, + ) + + @staticmethod + def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: + if isinstance(client, AsyncHTTPHandler): + return client + return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI) + async def async_realtime_calls_handler( self, api_base: str, @@ -6308,10 +6485,12 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - session_config: dict[str, object] | None = None, - extra_headers: dict[str, object] | None = None, - client: HTTPHandler | AsyncHTTPHandler | None = None, + session_config: Mapping[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | AsyncOpenAI | None = None, api_version: str | None = None, + translation: bool = False, + use_openai_sdk: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. @@ -6323,18 +6502,45 @@ class BaseLLMHTTPHandler: - sdp: the SDP offer (text) - session: JSON string with {"type": "realtime", "model": "...", ...} """ - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders.OPENAI, + session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally + **( + session_config or {} # mutable-ok: absent session configuration starts from an empty provider payload ) - else: - async_httpx_client = client + } + if "type" not in session_data: + session_data["type"] = "translation" if translation else "realtime" + if "model" not in session_data and model: + session_data["model"] = model + + sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + + if use_openai_sdk: + return await self._async_realtime_calls_sdk( + api_base=api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_text=sdp_text, + session_data=session_data, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=client, + translation=translation, + ) + + async_httpx_client: Final = self._get_realtime_async_http_client(client) if provider_config is not None: - url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) + url = ( + provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version) + if translation + else provider_config.get_realtime_calls_url( + api_base=api_base, model=model or "", api_version=api_version + ) + ) headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key) else: - url = f"{api_base.rstrip('/')}/v1/realtime/calls" + path: Final = "translations/calls" if translation else "calls" + url = f"{api_base.rstrip('/')}/v1/realtime/{path}" headers = { "Authorization": f"Bearer {openai_ephemeral_key}", } @@ -6342,14 +6548,8 @@ class BaseLLMHTTPHandler: if extra_headers: headers.update(extra_headers) - # Build multipart form data: sdp + session JSON - session_data: Final = session_config or {} - if "type" not in session_data: - session_data["type"] = "realtime" - if "model" not in session_data and model: - session_data["model"] = model - - sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + if translation: + headers["Content-Type"] = "application/sdp" files: Final = { "sdp": (None, sdp_text, "text/plain"), @@ -6367,12 +6567,14 @@ class BaseLLMHTTPHandler: ) try: - return await async_httpx_client.post( - url=url, - headers=headers, - files=files, - timeout=timeout, - ) + if translation: + return await async_httpx_client.post( + url=url, + headers=headers, + content=sdp_text, + timeout=timeout, + ) + return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout) except Exception as e: if provider_config is not None: raise self._handle_error( diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index bdc3a6c7908..c7801e8f201 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -5,8 +5,17 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ import ssl +from collections.abc import Mapping +from contextlib import AbstractAsyncContextManager +from types import MappingProxyType, TracebackType from typing import Any, Final, cast +from openai import AsyncOpenAI, omit +from openai.resources.realtime.realtime import ( + AsyncRealtimeConnection, + AsyncRealtimeConnectionManager, +) + from litellm._logging import _redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams @@ -22,6 +31,49 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..openai import OpenAIChatCompletion +class OpenAIRealtimeConnectionAdapter: + def __init__(self, connection: AsyncRealtimeConnection) -> None: + self._connection = connection + + async def send(self, message: str) -> None: + await self._connection.send_raw(message) + + async def recv(self, decode: bool = True) -> str | bytes: + message: Final = await self._connection.recv_bytes() + if decode: + return message.decode("utf-8") + return message + + async def close(self) -> None: + await self._connection.close() + + +class OpenAIRealtimeSDKConnectionManager: + def __init__( + self, + manager: AsyncRealtimeConnectionManager, + owned_client: AsyncOpenAI | None = None, + ) -> None: + self._manager = manager + self._owned_client = owned_client + + async def __aenter__(self) -> OpenAIRealtimeConnectionAdapter: + connection: Final = await self._manager.__aenter__() + return OpenAIRealtimeConnectionAdapter(connection) + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + await self._manager.__aexit__(exc_type, exc, traceback) + finally: + if self._owned_client is not None: + await self._owned_client.close() + + class OpenAIRealtime(OpenAIChatCompletion): """ Base handler for OpenAI-compatible realtime WebSocket connections. @@ -82,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion): return ssl_config - def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str: + def _construct_url(self, api_base: str, query_params: RealtimeQueryParams, realtime_mode: str = "realtime") -> str: """ Construct the backend websocket URL with all query parameters (including 'model'). """ @@ -92,7 +144,8 @@ class OpenAIRealtime(OpenAIChatCompletion): api_base = api_base.replace("http://", "ws://") url = URL(api_base) # Set the correct path - url = url.copy_with(path="/v1/realtime") + path: Final = "/v1/realtime/translations" if realtime_mode == "translation" else "/v1/realtime" + url = url.copy_with(path=path) # Include all query parameters including 'model' if query_params: url = url.copy_with(params=query_params) @@ -106,6 +159,46 @@ class OpenAIRealtime(OpenAIChatCompletion): """ return None + def _create_connection_manager( + self, + api_base: str, + api_key: str, + model: str, + query_params: RealtimeQueryParams, + headers: Mapping[str, str], + timeout: float | None, + realtime_mode: str, + ssl_config: object, + client: object | None, + url: str, + ) -> AbstractAsyncContextManager[object]: + import websockets + + if realtime_mode == "translation" or not isinstance(client, AsyncOpenAI): + return websockets.connect( + url, + additional_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=ssl_config, + **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), + ) + openai_client: Final = client + model_query: Final = query_params.get("model") + extra_query: Final = MappingProxyType({key: value for key, value in query_params.items() if key != "model"}) + sdk_model: Final = omit if query_params.get("intent") == "transcription" else model_query or model + sdk_connection_manager: Final = openai_client.realtime.connect( + model=sdk_model, + extra_query=extra_query, + extra_headers=headers, + websocket_connection_options={ # mutable-ok: OpenAI SDK forwards a mutable options mapping + "max_size": REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + **(MappingProxyType({"ssl": ssl_config}) if url.startswith("wss://") else MappingProxyType({})), + **(MappingProxyType({"open_timeout": timeout}) if timeout is not None else MappingProxyType({})), + }, + max_retries=0, + ) + return OpenAIRealtimeSDKConnectionManager(sdk_connection_manager) + async def async_realtime( self, model: str, @@ -118,6 +211,7 @@ class OpenAIRealtime(OpenAIChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", **kwargs: object, ): import websockets @@ -131,7 +225,7 @@ class OpenAIRealtime(OpenAIChatCompletion): # Use all query params if provided, else fallback to just model if query_params is None: query_params = {"model": model} - url: Final = self._construct_url(api_base, query_params) + url: Final = self._construct_url(api_base, query_params, realtime_mode=realtime_mode) try: # Get provider-specific SSL configuration @@ -156,15 +250,25 @@ class OpenAIRealtime(OpenAIChatCompletion): "complete_input_dict": {"query_params": query_params}, }, ) - async with websockets.connect( - url, - additional_headers=headers, - max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, - ssl=ssl_config, - ) as backend_ws: + connection_manager: Final = self._create_connection_manager( + api_base=api_base, + api_key=api_key, + model=model, + query_params=query_params, + headers=headers, + timeout=timeout, + realtime_mode=realtime_mode, + ssl_config=ssl_config, + client=client, + url=url, + ) + + async with connection_manager as backend_ws: realtime_streaming: Final = RealTimeStreaming( websocket, - cast(ClientConnection, backend_ws), + cast( # cast-ok: both SDK and websockets adapters implement the streaming connection interface + ClientConnection, backend_ws + ), logging_obj, model=model, user_api_key_dict=user_api_key_dict, @@ -173,6 +277,7 @@ class OpenAIRealtime(OpenAIChatCompletion): model if (query_params or {}).get("intent") == "transcription" else None ), event_normalizer=self._make_event_normalizer(), + translation_session=realtime_mode == "translation", ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py index 61dbf20397f..726a9545e11 100644 --- a/litellm/llms/openai/realtime/http_transformation.py +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -27,6 +27,18 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): base = base.removesuffix("/v1") return f"{base}/v1/realtime/transcription_sessions" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/client_secrets" + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/calls" + def validate_environment( self, headers: dict, diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index e7f8dd04516..c55ecf1e56c 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -14,7 +14,7 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): """ Get the supported OpenAI params for the `gpt-4o-transcribe` models """ - return [ + return [ # mutable-ok: base transcription interface requires a mutable supported-parameter list "language", "prompt", "response_format", @@ -37,3 +37,29 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): return AudioTranscriptionRequestData( data=data, ) + + +class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig): + def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: + return [ + "prompt", + "response_format", + "keywords", + "languages", + "stream", + ] + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, # mutable-ok: base transformation interface supplies a mutable request payload + litellm_params: dict, # mutable-ok: base transformation interface supplies mutable provider parameters + ) -> AudioTranscriptionRequestData: + data: Final = { # mutable-ok: OpenAI SDK consumes this multipart request mapping + "model": model, + "file": audio_file, + "response_format": "json", + **optional_params, + } + return AudioTranscriptionRequestData(data=data) diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 014251db821..ac57cc72290 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -11,6 +11,7 @@ from litellm.litellm_core_utils.core_helpers import set_provider_response_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, + sdk_compatible_transcription_request_data, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -31,11 +32,17 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout: float | httpx.Timeout, ): try: - raw_response = await openai_aclient.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) + sdk_data: Final = sdk_compatible_transcription_request_data(data) + if data.get("stream") is True: + stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout) + return {}, stream_response # mutable-ok: response headers use the existing mutable mapping contract + raw_response: Final = await openai_aclient.audio.transcriptions.with_raw_response.create( + **sdk_data, timeout=timeout + ) # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() + parsed_response: Final = raw_response.parse() - return headers, response + return headers, parsed_response except Exception as e: raise e @@ -46,7 +53,13 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout: float | httpx.Timeout, ): try: - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) + sdk_data: Final = sdk_compatible_transcription_request_data(data) + if data.get("stream") is True: + stream_response: Final = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout) + return None, stream_response + raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create( + **sdk_data, timeout=timeout + ) headers: Final = dict(raw_response.headers) response: Final = raw_response.parse() return headers, response @@ -125,6 +138,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): ) logging_obj.model_call_details["response_headers"] = headers + if data.get("stream") is True: + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -187,6 +203,8 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, ) logging_obj.model_call_details["response_headers"] = headers + if data.get("stream") is True: + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() else: diff --git a/litellm/main.py b/litellm/main.py index 6c85adf3ae8..a9fc32f6518 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,6 +37,8 @@ if TYPE_CHECKING: import dotenv import httpx import openai +from openai import AsyncStream, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel from typing_extensions import assert_never, overload @@ -7840,7 +7842,10 @@ async def amoderation( @client -async def atranscription(*args, **kwargs) -> TranscriptionResponse: +async def atranscription( + *args, # noqa: ANN002 # public SDK wrapper preserves positional call compatibility + **kwargs, # noqa: ANN003 # kwargs-ok: public SDK wrapper preserves keyword call compatibility +) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: """ Calls openai + azure whisper endpoints. @@ -7873,6 +7878,12 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: else: # Call the synchronous function using run_in_executor response = await loop.run_in_executor(None, func_with_context) + if kwargs.get("stream") is True and isinstance(response, AsyncStream): + if file is not None: + calculated_duration = calculate_request_duration(file) + if calculated_duration is not None: + response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field + return response if not isinstance(response, TranscriptionResponse): raise ValueError( f"Invalid response from transcription provider, expected TranscriptionResponse, but got {type(response)}" @@ -7885,9 +7896,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: if response is not None and not isinstance(response, Coroutine) and file is not None: existing_duration: Final = getattr(response, "duration", None) if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) - if calculated_duration is not None: - response._hidden_params["audio_transcription_duration"] = calculated_duration + sync_calculated_duration: Final = calculate_request_duration(file) + if sync_calculated_duration is not None: + response.set_audio_transcription_duration(sync_calculated_duration) return response except Exception as e: @@ -7901,16 +7912,58 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: ) +def _validate_gpt_transcription_request( + model: str, + custom_llm_provider: str, + language: str | None, + languages: Sequence[str] | None, + response_format: str | None, +) -> None: + model_cost_key: Final = next( + (key for key in (f"{custom_llm_provider}/{model}", model) if key in litellm.model_cost), None + ) + model_info: Final = ( + get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_cost_key is not None else None + ) + supported_endpoints: Final = model_info.get("supported_endpoints") if model_info is not None else None + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + if language is not None and languages is not None: + raise litellm.UnsupportedParamsError( + message="language and languages cannot be used together", + model=model, + llm_provider=custom_llm_provider, + ) + if supported_endpoints is not None and "/v1/audio/transcriptions" not in supported_endpoints: + raise litellm.UnsupportedParamsError( + message=f"{model} is available through the Realtime API, not file transcription", + model=model, + llm_provider=custom_llm_provider, + ) + if ( + provider_specific_entry is not None + and provider_specific_entry.get("transcription_json_only") == 1 + and response_format not in (None, "json") + ): + raise litellm.UnsupportedParamsError( + message=f"{model} only supports response_format='json'", + model=model, + llm_provider=custom_llm_provider, + ) + + @client def transcription( model: str, file: FileTypes, ## OPTIONAL OPENAI PARAMS ## language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: Literal["json", "text", "srt", "verbose_json", "vtt"] | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, temperature: int | None = None, # openai defaults this to 0 + stream: bool | None = None, ## LITELLM PARAMS ## user: str | None = None, timeout=600, # default to 10 minutes @@ -7920,7 +7973,11 @@ def transcription( max_retries: int | None = None, custom_llm_provider=None, **kwargs, -) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: +) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] +): """ Calls openai + azure whisper endpoints. @@ -7958,13 +8015,24 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key + _validate_gpt_transcription_request( + model=model, + custom_llm_provider=custom_llm_provider, + language=language, + languages=languages, + response_format=response_format, + ) + optional_params: Final = get_optional_params_transcription( model=model, language=language, + languages=languages, + keywords=keywords, prompt=prompt, response_format=response_format, timestamp_granularities=timestamp_granularities, temperature=temperature, + stream=stream, custom_llm_provider=custom_llm_provider, **non_default_params, ) @@ -7987,7 +8055,13 @@ def transcription( custom_llm_provider=custom_llm_provider, ) - response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None + response: ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | AsyncStream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + | None + ) = None provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model, @@ -8002,7 +8076,7 @@ def transcription( # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + azure_api_version: Final = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN") @@ -8021,7 +8095,7 @@ def transcription( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - api_version=api_version, + api_version=azure_api_version, azure_ad_token=azure_ad_token, max_retries=max_retries, litellm_params=litellm_params_dict, @@ -8155,11 +8229,12 @@ def transcription( # Store duration in _hidden_params for cost calculation without # exposing it in the response body (see sync path comment above). if response is not None and not isinstance(response, Coroutine): - existing_duration: Final = getattr(response, "duration", None) - if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) + calculated_duration: Final = calculate_request_duration(file) + if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - response._hidden_params["audio_transcription_duration"] = calculated_duration + response._litellm_audio_duration = calculated_duration # pyright: ignore[reportAttributeAccess] # SDK streams permit runtime metadata but do not type this field + elif getattr(response, "duration", None) is None and calculated_duration is not None: + response.set_audio_transcription_duration(calculated_duration) if response is None: raise ValueError("Unmapped provider passed in. Unable to get the response.") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d5490f5df9e..cac0aec3830 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,4 +1,264 @@ { + "azure/gpt-live-transcribe": { + "deprecation_date": "2028-02-01", + "input_cost_per_second": 0.000283333333333, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate": { + "deprecation_date": "2027-05-06", + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.0005666666666666667, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-transcribe": { + "deprecation_date": "2028-02-01", + "input_cost_per_second": 7.5e-05, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/audio/transcriptions", + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -6079,6 +6339,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-07-31", "input_cost_per_audio_token": 3.2e-05, @@ -6089,6 +6350,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6108,12 +6372,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-07-31", "input_cost_per_audio_token": 1e-05, @@ -6124,6 +6391,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6143,6 +6413,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -6322,6 +6594,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -26739,9 +27014,9 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -35718,6 +35993,9 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -35774,6 +36052,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -35811,6 +36091,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -50035,9 +50317,9 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -60070,6 +60352,9 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_only": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -60082,7 +60367,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -60100,7 +60386,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -60126,7 +60413,13 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], "supported_modalities": [ "audio" ], @@ -60135,7 +60428,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, @@ -70825,6 +71119,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -70865,66 +71162,6 @@ "supports_audio_output": true, "supports_function_calling": true }, - "azure/gpt-live-transcribe": { - "deprecation_date": "2028-02-01", - "input_cost_per_second": 0.000283333333333, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "audio_transcription", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/realtime", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true - }, - "azure/gpt-transcribe": { - "deprecation_date": "2028-02-01", - "input_cost_per_second": 7.5e-05, - "litellm_provider": "azure", - "mode": "audio_transcription", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true - }, - "azure/gpt-realtime-translate": { - "deprecation_date": "2027-05-06", - "input_cost_per_second": 0.000566666666667, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_modalities": [ - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true - }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, "litellm_provider": "aihubmix", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 5be87a8bf4d..ed09149904e 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -31,6 +31,20 @@ def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], return _register +def _include_router_prepend(attr_name: str = "router") -> Callable[["FastAPI", object], None]: + def _register(app: "FastAPI", module: object) -> None: + router: Final = app.router + route_count: Final = len(router.routes) + app.include_router(getattr(module, attr_name)) + new_routes: Final = router.routes[route_count:] + router.routes[:] = [ # mutable-ok: FastAPI route registration requires in-place list replacement + *new_routes, + *router.routes[:route_count], + ] + + return _register + + def _mount_app(prefix: str, attr_name: str = "app") -> Callable[["FastAPI", object], None]: def _register(app: "FastAPI", module: object) -> None: app.mount(path=prefix, app=getattr(module, attr_name)) @@ -225,6 +239,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( name="realtime", module_path="litellm.proxy.realtime_endpoints.endpoints", path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"), + register_fn=_include_router_prepend(), ), LazyFeature( name="anthropic_passthrough", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index c4ff4b415dd..44e42c52673 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -27410,6 +27410,51 @@ ] } }, + "/openai/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "llm_passthrough" + ] + } + }, + "/openai/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "llm_passthrough" + ] + } + }, "/openai/v1/responses": { "post": { "description": "Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses\n\nSupports background mode with polling_via_cache for partial response retrieval.\nWhen background=true and polling_via_cache is enabled, returns a polling_id immediately\nand streams the response in the background, updating Redis cache.\n\n```bash\n# Normal request\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer sk-1234\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\"\n}'\n\n# Background request with polling\ncurl -X POST http://localhost:4000/v1/responses -H \"Content-Type: application/json\" -H \"Authorization: Bearer sk-1234\" -d '{\n \"model\": \"gpt-4o\",\n \"input\": \"Tell me about AI\",\n \"background\": true\n}'\n```", @@ -46491,6 +46536,51 @@ ] } }, + "/openai/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_openai_v1_realtime_translations_calls_post_2", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/openai/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post_2", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/realtime/calls": { "post": { "operationId": "proxy_realtime_calls_realtime_calls_post", @@ -46563,6 +46653,51 @@ ] } }, + "/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/v1/realtime/calls": { "post": { "operationId": "proxy_realtime_calls_v1_realtime_calls_post", @@ -46634,6 +46769,51 @@ "realtime" ] } + }, + "/v1/realtime/translations/calls": { + "post": { + "operationId": "proxy_realtime_calls_v1_realtime_translations_calls_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/translations/client_secrets": { + "post": { + "operationId": "create_realtime_client_secret_v1_realtime_translations_client_secrets_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d9fb053035b..58eea283e4f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -420,6 +420,9 @@ class LiteLLMRoutes(enum.Enum): "/realtime?{model}", "/v1/realtime?{model}", "/openai/v1/realtime?{model}", + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", # realtime (GA WebRTC HTTP routes) "/realtime/client_secrets", "/v1/realtime/client_secrets", @@ -430,6 +433,12 @@ class LiteLLMRoutes(enum.Enum): "/realtime/transcription_sessions", "/v1/realtime/transcription_sessions", "/openai/v1/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", # responses API "/responses", "/v1/responses", @@ -2631,6 +2640,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): admission_queue_timeout_seconds: float = Field( 1.0, gt=0, description="maximum time a request waits for a worker slot" ) + allow_non_billable_realtime_protocols: bool = Field( + False, + description="Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement", + ) plugins: list[PluginConfig] | None = Field( None, description="external services registered as embeddable UI plugins" ) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 97de2488b8d..2b9e776e09b 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -136,7 +136,9 @@ ProxyRouteType: TypeAlias = Literal[ "_arealtime", "_aresponses_websocket", "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "aget_responses", "adelete_responses", "acancel_responses", diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1c2bd7ea217..37805ff0459 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -399,14 +399,18 @@ async def get_form_data(request: Request) -> dict[str, Any]: Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` """ form: Final = await request.form() - parsed_form_data: Final[dict[str, Any]] = {} - for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat - if key.endswith("[]"): - clean_key = key[:-2] - parsed_form_data.setdefault(clean_key, []).append(value) - else: - parsed_form_data[key] = value - return parsed_form_data + form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items()) + array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]")) + normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items) + normalized_keys: Final = frozenset(key for key, _ in normalized_items) + return { # mutable-ok: request parsers expose a mutable form-data mapping to endpoint handlers + key: [ # mutable-ok: repeated multipart values follow the established mutable list contract + value for item_key, value in normalized_items if item_key == key + ] + if key in array_keys + else next(value for item_key, value in reversed(normalized_items) if item_key == key) + for key in normalized_keys + } async def convert_upload_files_to_file_data( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7cd116c23d7..068319965f7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -50,6 +50,7 @@ from typing import ( import anyio import websockets import websockets.exceptions +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, Json, JsonValue, TypeAdapter, ValidationError from pydantic.fields import FieldInfo, PydanticUndefined from typing_extensions import NotRequired, ReadOnly, assert_never @@ -12442,7 +12443,11 @@ async def audio_transcriptions( try: # Use orjson to parse JSON data, orjson speeds up requests significantly form_data: Final = await get_form_data(request) - data = {key: value for key, value in form_data.items() if key != "file"} | data + data = { + key: value is True or str(value).lower() in ("1", "true") if key == "stream" else value + for key, value in form_data.items() + if key != "file" + } | data # Include original request and headers in the data data = await add_litellm_data_to_request( @@ -12495,6 +12500,12 @@ async def audio_transcriptions( call_type="transcription", ) + if data.get("stream") is True and ProxyLogging.has_post_call_guardrails(data, llm_router): + raise HTTPException( + detail="Streaming transcription does not support output guardrails. Use stream=false.", + status_code=status.HTTP_400_BAD_REQUEST, + ) + ## ROUTE TO CORRECT ENDPOINT ## llm_call: Final = await route_request( data=data, @@ -12508,6 +12519,29 @@ async def audio_transcriptions( finally: file_object.close() # close the file read in by io library + if data.get("stream") is True: + if not hasattr(response, "__aiter__"): + raise TypeError(f"Streaming transcription returned {type(response).__name__}, expected an async stream") + stream_response: Final = cast(AsyncIterator[TranscriptionStreamEvent], response) + + async def transcription_event_stream( + stream: AsyncIterator[TranscriptionStreamEvent], + ) -> AsyncGenerator[str, None]: + try: + async for event in stream: + yield f"data: {event.model_dump_json()}\n\n" + finally: + close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None) + if callable(close): + close_result: Final = close() + if inspect.isawaitable(close_result): + await close_result + + return StreamingResponse( + transcription_event_stream(stream_response), + media_type="text/event-stream", + ) + ### ALERTING ### asyncio.create_task(proxy_logging_obj.update_request_status(litellm_call_id=litellm_call_id, status="success")) @@ -12674,9 +12708,39 @@ async def _reject_realtime_session( await _release_realtime_max_parallel_slot(user_api_key_dict) +def _resolve_realtime_route_model( + model: str | None, + intent: str | None, + is_translation: bool, +) -> str | None: + if model is not None: + return model + if is_translation: + return "gpt-realtime-translate" + if intent == "transcription": + return "gpt-realtime-whisper" + return None + + +def _resolve_realtime_upstream_query_model( + model: str | None, + intent: str | None, + is_translation: bool, + route_model: str, +) -> str | None: + if intent == "transcription": + return None + if is_translation: + return route_model + return model + + @app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") +@app.websocket("/openai/v1/realtime/translations") +@app.websocket("/v1/realtime/translations") +@app.websocket("/realtime/translations") async def realtime_websocket_endpoint( websocket: WebSocket, model: str | None = fastapi.Query(None, description="The model to use for the websocket connection."), @@ -12694,15 +12758,13 @@ async def realtime_websocket_endpoint( if requested_protocols: accept_kwargs["subprotocol"] = requested_protocols[0] - route_model = model + is_translation: Final = websocket.url.path.endswith("/realtime/translations") + route_model: Final = _resolve_realtime_route_model(model, intent, is_translation) if route_model is None: - if intent == "transcription": - route_model = "gpt-realtime-whisper" - else: - await _reject_realtime_session( - websocket, user_api_key_dict, code=1008, reason="model query parameter is required" - ) - return + await _reject_realtime_session( + websocket, user_api_key_dict, code=1008, reason="model query parameter is required" + ) + return assert route_model is not None try: await can_key_call_resolved_model( @@ -12718,12 +12780,24 @@ async def realtime_websocket_endpoint( await websocket.accept(**accept_kwargs) # Only use explicit parameters, not all query params - query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent))) + query_model: Final = _resolve_realtime_upstream_query_model( + model=model, + intent=intent, + is_translation=is_translation, + route_model=route_model, + ) + query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys + RealtimeQueryParams, + dict( # mutable-ok: downstream realtime routing normalizes this request-scoped query mapping + _realtime_query_params_template(query_model, intent) + ), + ) data: dict[str, object] = { "model": route_model, "websocket": websocket, "query_params": query_params, # Only explicit params + "realtime_mode": "translation" if is_translation else "realtime", } # Pass guardrails into data so pre-call guardrail processing picks them up diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index be2ac2ff33e..f358111835b 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -2,6 +2,7 @@ import json import time +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -37,7 +38,21 @@ router: Final = APIRouter() _REALTIME_TOKEN_VERSION: Final = "realtime_v1" _DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview" _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" -_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription") +_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation") +_NON_BILLABLE_REALTIME_PROTOCOL_SETTING: Final = "allow_non_billable_realtime_protocols" +_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE: Final = ( + "Realtime WebRTC endpoints are disabled because provider usage bypasses LiteLLM billing. " + "Set general_settings.allow_non_billable_realtime_protocols to true to opt in" +) + + +def _enforce_non_billable_realtime_protocol_gate(general_settings: Mapping[str, object]) -> None: + if general_settings.get(_NON_BILLABLE_REALTIME_PROTOCOL_SETTING) is True: + return + raise HTTPException( + status_code=http_status.HTTP_403_FORBIDDEN, + detail=_NON_BILLABLE_REALTIME_PROTOCOL_MESSAGE, + ) def _coerce_realtime_session_type(session_type: str | None) -> str: @@ -120,19 +135,47 @@ def _set_transcription_model_on_session( } +async def _authorize_and_bind_nested_transcription_models( + session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization + user_api_key_dict: UserAPIKeyAuth, + llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list + llm_router: "Router | None", +) -> None: + nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data)) + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + if nested_models: + _set_transcription_model_on_session( + session=session_data, + model=nested_models[0], + ) + + async def _prepare_client_secret_session( req: RealtimeClientSecretRequest, user_api_key_dict: UserAPIKeyAuth, llm_model_list: list | None, llm_router: "Router | None", + forced_session_type: str | None = None, ) -> tuple[str, dict | None, str]: - session_type: Final = _coerce_realtime_session_type(req.session.type if req.session else None) - session_data: Final[dict | None] = req.session.model_dump(exclude_none=True) if req.session else None + requested_session_type: Final = req.session.type if req.session else None + if forced_session_type is None and requested_session_type == "translation": + raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint") + session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type) + session_data: Final[dict | None] = ( + req.session.model_dump(exclude_none=True) if req.session else ({} if session_type == "translation" else None) + ) if session_data is not None: session_data["type"] = session_type session_model: Final = req.session.model if req.session else None - model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL + default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL + model: str = session_model or req.model or default_model if session_type != "transcription": await can_key_call_resolved_model( model=model, @@ -140,6 +183,15 @@ async def _prepare_client_secret_session( llm_model_list=llm_model_list, llm_router=llm_router, ) + if session_data is not None: + session_data["model"] = model + if session_type == "translation": + await _authorize_and_bind_nested_transcription_models( + session_data=session_data, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) return model, session_data, session_type transcription_model_candidates: Final = _transcription_model_candidates_from_session(session_data or {}) @@ -228,6 +280,21 @@ def _decode_realtime_token_payload( dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/openai/v1/realtime/translations/client_secrets", + dependencies=[Depends(user_api_key_auth)], # mutable-ok: FastAPI decorator requires dependency lists + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) async def create_realtime_client_secret( request: Request, fastapi_response: Response, @@ -245,16 +312,19 @@ async def create_realtime_client_secret( version, ) + _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await _read_request_body(request=request) req: Final = RealtimeClientSecretRequest(**body) + is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path model, session_data, session_type = await _prepare_client_secret_session( req=req, user_api_key_dict=user_api_key_dict, llm_model_list=llm_model_list, llm_router=llm_router, + forced_session_type="translation" if is_translation_request else None, ) data = {"model": model} @@ -278,17 +348,20 @@ async def create_realtime_client_secret( proxy_config=proxy_config, ) + call_type: Final = ( + "acreate_realtime_translation_client_secret" if is_translation_request else "acreate_realtime_client_secret" + ) data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=data, - call_type="acreate_realtime_client_secret", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/client_secrets (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="acreate_realtime_client_secret", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -371,6 +444,18 @@ async def create_realtime_client_secret( "/openai/v1/realtime/calls", tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) +@router.post( + "/openai/v1/realtime/translations/calls", + tags=["realtime"], # mutable-ok: FastAPI decorator requires tag lists +) async def proxy_realtime_calls( request: Request, fastapi_response: Response, @@ -396,6 +481,7 @@ async def proxy_realtime_calls( media_type="application/json", ) + is_translation_request: Final = "/realtime/translations/calls" in request.url.path encrypted_token: Final = auth_header.removeprefix("Bearer ").strip() decrypted_token_value: Final = decrypt_value_helper( value=encrypted_token, @@ -408,26 +494,38 @@ async def proxy_realtime_calls( media_type="application/json", ) + _enforce_non_billable_realtime_protocol_gate(general_settings) sdp_body: Final[bytes] = await request.body() decoded_payload: Final = _decode_realtime_token_payload(decrypted_token_value) if decoded_payload is not None: # Check token expiry expires_at: Final = decoded_payload.get("expires_at") - if expires_at is not None and isinstance(expires_at, int): - if time.time() > expires_at: - return Response( - content=json.dumps({"error": "Token has expired"}), - status_code=http_status.HTTP_401_UNAUTHORIZED, - media_type="application/json", - ) + if isinstance(expires_at, int) and time.time() > expires_at: + return Response( + content=json.dumps({"error": "Token has expired"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decoded_payload.get("ephemeral_key", "") model = decoded_payload.get("model_id") or request.query_params.get("model") or _DEFAULT_REALTIME_MODEL user_id = decoded_payload.get("user_id") or None team_id = decoded_payload.get("team_id") or None - session_type = _coerce_realtime_session_type(decoded_payload.get("session_type")) + raw_session_type: Final = decoded_payload.get("session_type") + session_type = _coerce_realtime_session_type(raw_session_type) + if is_translation_request != (raw_session_type == "translation"): + return Response( + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) else: - # Backward compatibility: older tokens contained only encrypted upstream key. + if is_translation_request: + return Response( + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decrypted_token_value model = request.query_params.get("model", _DEFAULT_REALTIME_MODEL) user_id = None @@ -471,17 +569,18 @@ async def proxy_realtime_calls( proxy_config=proxy_config, ) + call_type: Final = "arealtime_translation_calls" if is_translation_request else "arealtime_calls" data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=minimal_auth, data=data, - call_type="arealtime_calls", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/calls (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="arealtime_calls", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -557,6 +656,7 @@ async def create_realtime_transcription_session( version, ) + _enforce_non_billable_realtime_protocol_gate(general_settings) data: dict = {} try: body: Final = await _read_request_body(request=request) @@ -571,6 +671,13 @@ async def create_realtime_transcription_session( ) transcription_session: Final = {k: v for k, v in body.items() if k != "model"} + await _authorize_and_bind_nested_transcription_models( + session_data=transcription_session, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + _set_transcription_model_on_session(session=transcription_session, model=model, create_if_missing=True) data = {"model": model, "transcription_session": transcription_session} data = await add_litellm_data_to_request( diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 536c58df65a..399ecc19d27 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -342,7 +342,9 @@ RouteType = Literal[ "alist_input_items", "_arealtime", # private function for realtime API "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "acreate_realtime_transcription_session", "_aresponses_websocket", # private function for responses WebSocket mode "aimage_edit", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1fca50e24c9..c55b2a6cff3 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2876,6 +2876,17 @@ class ProxyLogging: def has_post_call_response_headers_callbacks() -> bool: return ProxyLogging._callback_capabilities().has_post_call_response_headers + @staticmethod + def has_post_call_guardrails(request_data: Mapping[str, object], llm_router: Router | None) -> bool: + guardrail_data: Final = _check_and_merge_model_level_guardrails( + data=dict(request_data), llm_router=llm_router, trust_client_model_info=False + ) + guardrails, _ = _partition_post_call_callbacks() + return bool(pipeline_managed_guardrail_names(guardrail_data, "post_call")) or any( + guardrail.should_run_guardrail(data=guardrail_data, event_type=GuardrailEventHooks.post_call) + for guardrail in guardrails + ) + @staticmethod def has_streaming_callbacks() -> bool: caps: Final = ProxyLogging._callback_capabilities() diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 28814741852..9ac86177fd7 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -6,6 +6,8 @@ from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast +import httpx + import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, @@ -14,6 +16,7 @@ from litellm.constants import ( request_timeout, ) from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.xai.common_utils import XAIModelInfo @@ -47,8 +50,6 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket - from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig - azure_realtime: Final = AzureOpenAIRealtime() openai_realtime: Final = OpenAIRealtime() bedrock_realtime: Final = BedrockRealtime() @@ -70,6 +71,32 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: + if session.get("type") == "transcription": + audio = session.get("audio") + audio = audio if isinstance(audio, dict) else {} # mutable-ok: nested session model is rebuilt locally + audio_input = audio.get("input") + audio_input = ( + audio_input if isinstance(audio_input, dict) else {} # mutable-ok: nested session model is rebuilt locally + ) + transcription = audio_input.get("transcription") + transcription = ( + transcription + if isinstance(transcription, dict) + else {} # mutable-ok: nested session model is rebuilt locally + ) + return { # mutable-ok: provider routing requires an independently mutable session payload + **session, + "audio": { # mutable-ok: provider routing rebuilds nested audio configuration + **audio, + "input": { # mutable-ok: provider routing rebuilds nested input configuration + **audio_input, + "transcription": { # mutable-ok: resolved deployment replaces only the transcription model + **transcription, + "model": model_name, + }, + }, + }, + } if "model" not in session: return session return {**session, "model": model_name} @@ -97,10 +124,6 @@ def _get_realtime_http_provider_config( Uses ProviderConfigManager so each provider keeps its credential-resolution and URL-construction logic in its own transformation class. """ - from litellm.llms.base_llm.realtime.http_transformation import ( - BaseRealtimeHTTPConfig, - ) - provider_config: BaseRealtimeHTTPConfig | None = None if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_realtime_http_config( @@ -124,6 +147,21 @@ def _get_realtime_http_provider_config( return provider_config, resolved_api_base.rstrip("/"), resolved_api_key +def _get_realtime_http_extra_headers( + provider_config: BaseRealtimeHTTPConfig | None, + litellm_params: GenericLiteLLMParams, + resolved_api_key: str, + extra_headers: Mapping[str, object] | None, +) -> Mapping[str, object] | None: + if provider_config is None: + return extra_headers + return provider_config.get_extra_headers( + litellm_params=litellm_params, + api_key=resolved_api_key, + extra_headers=extra_headers, + ) + + @wrapper_client async def acreate_realtime_client_secret( model: str | None = None, @@ -137,30 +175,48 @@ async def acreate_realtime_client_secret( session=RealtimeSessionConfig.model_validate(session) if session else None, expires_after=RealtimeExpiresAfter.model_validate(expires_after) if expires_after else None, ) - model_name = (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview" + transcription_model: Final = ( + req.session.audio.input.transcription.model + if req.session is not None + and req.session.audio is not None + and req.session.audio.input is not None + and req.session.audio.input.transcription is not None + else None + ) + provider_qualified_model: Final = ( + req.model + if req.model is not None + and "/" in req.model + and req.model.split("/", 1)[0] in LlmProviders._member_map_.values() + else None + ) + requested_model_name: Final = ( + provider_qualified_model + or transcription_model + or (req.session.model if req.session is not None else None) + or req.model + or "gpt-4o-realtime-preview" + ) litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj") litellm_params: Final = GenericLiteLLMParams(**kwargs) - ( - model_name, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = get_llm_provider( - model=model_name, + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) - ( - provider_config, - resolved_api_base, - resolved_api_key, - ) = _get_realtime_http_provider_config( + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( custom_llm_provider=custom_llm_provider, dynamic_api_base=dynamic_api_base, dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -171,6 +227,11 @@ async def acreate_realtime_client_secret( request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) if isinstance(request_data.get("session"), dict): request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + elif req.model is not None: + request_data["session"] = { # mutable-ok: OpenAI SDK consumes this request-scoped session payload + "type": "realtime", + "model": model_name, + } return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=resolved_api_key, @@ -179,9 +240,86 @@ async def acreate_realtime_client_secret( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", + ) + + +@wrapper_client +async def acreate_realtime_translation_client_secret( + model: str | None = None, + session: Mapping[str, Any] | None = None, + expires_after: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options +) -> httpx.Response: + requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate" + session_config: Final = RealtimeSessionConfig.model_validate( + { # mutable-ok: Pydantic validates this request-scoped translation session payload + **(session or {}), + "type": "translation", + "model": requested_model_name, + } + ) + req: Final = RealtimeClientSecretRequest( + model=requested_model_name, + session=session_config, + expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None, + ) + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ # mutable-ok: logging owns a mutable request metadata payload + "expires_after": expires_after, + "session": session, + }, + litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload + "api_base": resolved_api_base + }, + custom_llm_provider=custom_llm_provider, + ) + request_data: Final = req.model_dump( + exclude_none=True, + exclude={"model"}, # mutable-ok: Pydantic requires a mutable field-exclusion set + ) + request_data["session"] = _with_resolved_session_model(request_data["session"], model_name) + request_data["session"].pop("type", None) + return await base_llm_http_handler.async_realtime_translation_client_secret_handler( + api_base=resolved_api_base, + api_key=resolved_api_key, + request_data=request_data, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + extra_headers=resolved_extra_headers, + client=kwargs.get("client"), + api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", ) @@ -229,6 +367,12 @@ async def acreate_realtime_transcription_session( dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -251,7 +395,7 @@ async def acreate_realtime_transcription_session( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, ) @@ -307,6 +451,70 @@ async def arealtime_calls( extra_headers=kwargs.get("extra_headers"), client=kwargs.get("client"), api_version=litellm_params.api_version, + use_openai_sdk=custom_llm_provider == "openai", + ) + + +@wrapper_client +async def arealtime_translation_calls( + openai_ephemeral_key: str, + sdp_body: bytes, + model: str | None = None, + session: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs, # noqa: ANN003 # kwargs-ok: public client wrapper forwards provider-specific options +) -> httpx.Response: + requested_model_name: Final = model or "gpt-realtime-translate" + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + session_config: Final = _with_resolved_session_model( + { # mutable-ok: provider routing requires an independently mutable session payload + **(session or {}), + "type": "translation", + "model": model_name, + }, + model_name, + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ # mutable-ok: logging owns a mutable request metadata payload + "realtime_translation_calls": True, + "session": session_config, + }, + litellm_params={ # mutable-ok: logging owns a mutable provider metadata payload + "api_base": resolved_api_base + }, + custom_llm_provider=custom_llm_provider, + ) + return await base_llm_http_handler.async_realtime_calls_handler( + api_base=resolved_api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_body=sdp_body, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + session_config=session_config, + extra_headers=kwargs.get("extra_headers"), + client=kwargs.get("client"), + api_version=litellm_params.api_version, + translation=True, + use_openai_sdk=custom_llm_provider == "openai", ) @@ -356,6 +564,7 @@ async def _arealtime( client: object | None = None, timeout: float | None = None, query_params: RealtimeQueryParams | None = None, + realtime_mode: str = "realtime", **kwargs, ): """ @@ -423,6 +632,9 @@ async def _arealtime( api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") # set API KEY api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_API_KEY") + resolved_azure_ad_token = azure_ad_token or litellm_params.azure_ad_token + if not api_key and not resolved_azure_ad_token: + resolved_azure_ad_token = get_azure_ad_token(litellm_params) api_version = api_version or litellm_params.api_version or "2024-10-01-preview" @@ -432,10 +644,11 @@ async def _arealtime( or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") ) realtime_protocol: Final = azure_realtime_protocol_for_client( - configured_realtime_protocol, query_params=query_params, websocket=websocket - ) - resolved_azure_ad_token: Final = ( - None if api_key else get_azure_ad_token(GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token)) + configured_realtime_protocol, + model=model, + query_params=query_params, + realtime_mode=realtime_mode, + websocket=websocket, ) await azure_realtime.async_realtime( model=model, @@ -449,6 +662,7 @@ async def _arealtime( logging_obj=litellm_logging_obj, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), litellm_metadata=_build_litellm_metadata(kwargs), ) @@ -463,9 +677,10 @@ async def _arealtime( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - client=None, + client=client, timeout=timeout, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), litellm_metadata=_build_litellm_metadata(kwargs), ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 12bc9adbac8..733eaf0d070 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -196,6 +196,8 @@ _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( "insufficient_quota": 429, "vector_store_timeout": 504, "invalid_prompt": 400, + "data_residency_mismatch": 400, + "bio_policy": 400, "invalid_image": 400, "invalid_image_format": 400, "invalid_base64_image": 400, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 9b0d259eb8a..9f7d96f5dbc 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -1188,12 +1188,14 @@ class ResponseAPILoggingUtils: else: prompt_tokens_details = PromptTokensDetailsWrapper( cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None), + cached_tokens_details=getattr( + response_api_usage.input_tokens_details, + "cached_tokens_details", + None, + ), audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None), text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None), image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None), - cached_tokens_details=getattr( - response_api_usage.input_tokens_details, "cached_tokens_details", None - ), video_tokens=getattr(response_api_usage.input_tokens_details, "video_tokens", None), cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None), web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None), diff --git a/litellm/router.py b/litellm/router.py index cfed5dc81c0..9dce9ebe2c8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1907,6 +1907,14 @@ class Router: self.acreate_realtime_transcription_session = self.factory_function( litellm.acreate_realtime_transcription_session, call_type="acreate_realtime_transcription_session" ) + self.acreate_realtime_translation_client_secret = self.factory_function( + litellm.acreate_realtime_translation_client_secret, + call_type="acreate_realtime_translation_client_secret", + ) + self.arealtime_translation_calls = self.factory_function( + litellm.arealtime_translation_calls, + call_type="arealtime_translation_calls", + ) self._aresponses_websocket = self.factory_function( litellm._aresponses_websocket, call_type="_aresponses_websocket" ) @@ -6568,6 +6576,8 @@ class Router: "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", "_aresponses_websocket", "acreate_fine_tuning_job", "acancel_fine_tuning_job", @@ -6829,6 +6839,8 @@ class Router: "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 99ab5920c4f..3db12eb7015 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1166,9 +1166,12 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]] OpenAIAudioTranscriptionOptionalParams = Literal[ "language", + "languages", + "keywords", "prompt", "temperature", "response_format", + "stream", "timestamp_granularities", "include", ] @@ -2320,6 +2323,17 @@ class OpenAIRealtimeResponseUsage(TypedDict): output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] +class OpenAIRealtimeTranslationDurationUsage(TypedDict): + type: ReadOnly[Literal["duration"]] + output_seconds: NotRequired[ReadOnly[float]] + input_seconds: NotRequired[ReadOnly[float]] + + +class OpenAIRealtimeTranslationClosedEvent(TypedDict): + type: ReadOnly[Literal["session.closed"]] + usage: ReadOnly[OpenAIRealtimeTranslationDurationUsage] + + class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" # Beta delta event names @@ -2362,6 +2376,7 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeInputAudioTranscriptionCompleted | OpenAIRealtimeTranscriptionSessionCreated | OpenAIRealtimeErrorEvent + | OpenAIRealtimeTranslationClosedEvent ) OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents] diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 855cccc8ddd..f6ab5fc31db 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,6 +1,7 @@ -from typing import Any, Literal +from collections.abc import Mapping, Sequence +from typing import Any, Final, Literal -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from typing_extensions import ReadOnly, TypedDict from .llms.openai import ( @@ -10,6 +11,7 @@ from .llms.openai import ( ) ALL_DELTA_TYPES = Literal["text", "audio"] +RealtimeSessionType: Final = Literal["realtime", "transcription", "translation"] class RealtimeResponseTransformInput(TypedDict): @@ -64,6 +66,41 @@ class RealtimeExpiresAfter(BaseModel): seconds: int | None = None +class RealtimeAudioTranscriptionConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + model: str | None = None + delay: Literal["minimal", "low", "medium", "high", "xhigh"] | None = None + keywords: Sequence[str] | None = None + language: str | None = None + languages: Sequence[str] | None = None + prompt: str | None = None + + +class RealtimeAudioInputConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + format: str | Mapping[str, object] | None = None + noise_reduction: Mapping[str, object] | None = None + transcription: RealtimeAudioTranscriptionConfig | None = None + turn_detection: Mapping[str, object] | None = None + + +class RealtimeAudioOutputConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + format: str | Mapping[str, object] | None = None + language: str | None = None + voice: str | Mapping[str, object] | None = None + + +class RealtimeSessionAudioConfig(BaseModel): + model_config = ConfigDict(extra="allow") + + input: RealtimeAudioInputConfig | None = None + output: RealtimeAudioOutputConfig | None = None + + class RealtimeSessionConfig(BaseModel): """ Session configuration nested inside the client_secrets request body. @@ -75,10 +112,10 @@ class RealtimeSessionConfig(BaseModel): model_config = {"extra": "allow"} - type: str | None = None + type: RealtimeSessionType | None = None model: str | None = None instructions: str | None = None - audio: dict[str, object] | None = None + audio: RealtimeSessionAudioConfig | None = None include: list[str] | None = None max_output_tokens: int | str | None = None output_modalities: list[str] | None = None @@ -132,12 +169,15 @@ class RealtimeTranscriptionSessionRequest(BaseModel): # LiteLLM-only routing hint — stripped before forwarding upstream. model: str | None = None input_audio_transcription: dict[str, Any] | None = None + audio: RealtimeSessionAudioConfig | None = None def resolved_model(self) -> str | None: if self.model: return self.model if self.input_audio_transcription: return self.input_audio_transcription.get("model") + if self.audio and self.audio.input and self.audio.input.transcription: + return self.audio.input.transcription.model return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b9be8b6858e..30c948b1f78 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -464,6 +464,11 @@ class CallTypes(str, Enum): asearch = "asearch" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" + acreate_realtime_client_secret = "acreate_realtime_client_secret" + arealtime_calls = "arealtime_calls" + acreate_realtime_transcription_session = "acreate_realtime_transcription_session" + acreate_realtime_translation_client_secret = "acreate_realtime_translation_client_secret" + arealtime_translation_calls = "arealtime_translation_calls" create_batch = "create_batch" acreate_batch = "acreate_batch" aretrieve_batch = "aretrieve_batch" @@ -692,10 +697,18 @@ CallTypesLiteral = Literal[ "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ] # Mapping of API routes to their corresponding call types API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { + "/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/openai/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/openai/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), # Chat Completions "/chat/completions": [CallTypes.acompletion, CallTypes.completion], "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], @@ -1008,9 +1021,12 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API - "/realtime": [CallTypes.arealtime], - "/v1/realtime": [CallTypes.arealtime], - "/openai/v1/realtime": [CallTypes.arealtime], + "/realtime": (CallTypes.arealtime,), + "/v1/realtime": (CallTypes.arealtime,), + "/openai/v1/realtime": (CallTypes.arealtime,), + "/realtime/translations": (CallTypes.arealtime,), + "/v1/realtime/translations": (CallTypes.arealtime,), + "/openai/v1/realtime/translations": (CallTypes.arealtime,), # Provider-specific routes "/anthropic/v1/messages": [CallTypes.anthropic_messages], # Google GenAI routes @@ -1735,6 +1751,8 @@ class PromptTokensDetailsWrapper( image_tokens: int | None = None """Image tokens sent to the model.""" + cached_tokens_details: CachedTokensDetails | None = None + video_tokens: int | None = None """Video tokens sent to the model.""" @@ -2719,18 +2737,26 @@ class TranscriptionUsageTokensObject(BaseModel): input_tokens: int output_tokens: int total_tokens: int - input_token_details: TranscriptionUsageInputTokenDetailsObject + input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None + + +class TranscriptionDetectedLanguage(BaseModel): + code: str class TranscriptionResponse(OpenAIObject): text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None + languages: Sequence[TranscriptionDetectedLanguage] | None = None _hidden_params: dict = {} _response_headers: dict | None = None - def __init__(self, text=None) -> None: - super().__init__(text=text) + def __init__(self, text=None, usage=None, languages=None, **kwargs) -> None: # noqa: ANN003 # OpenAI-compatible response accepts provider extension fields + super().__init__(text=text, usage=usage, languages=languages, **kwargs) + + def set_audio_transcription_duration(self, duration: float) -> None: + self._hidden_params["audio_transcription_duration"] = duration def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/litellm/utils.py b/litellm/utils.py index 13a46840431..08dad521581 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1279,6 +1279,8 @@ def function_setup( applied_guardrails=applied_guardrails, supports_correlation_logging=is_async_call, ) + if logging_obj is None: + raise RuntimeError("LiteLLM logging initialization returned no logger") ## check if metadata is passed in litellm_params: Final[dict[str, object]] = {"api_base": ""} @@ -1797,6 +1799,12 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.transcription.value and isinstance(result, openai.Stream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + wrap_transcription_stream, + ) + + result = wrap_transcription_stream(result, logging_obj, start_time) # RETURN RESULT update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata update_response_metadata( @@ -2095,6 +2103,12 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.atranscription.value and isinstance(result, openai.AsyncStream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + wrap_transcription_stream, + ) + + result = wrap_transcription_stream(result, logging_obj, start_time) _update_response_metadata( result=result, logging_obj=logging_obj, @@ -3470,19 +3484,23 @@ def get_optional_params_transcription( model: str, custom_llm_provider: str, language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: str | None = None, temperature: int | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, + stream: bool | None = None, drop_params: bool | None = None, **kwargs, ): from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS # retrieve all parameters passed to the function - passed_params: Final = locals() + passed_params: Final = locals().copy() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") + passed_params.pop("model") custom_llm_provider = passed_params.pop("custom_llm_provider") passed_params.pop("drop_params") drop_params = normalize_drop_params(drop_params) @@ -3492,10 +3510,13 @@ def get_optional_params_transcription( default_params: Final = { "language": None, + "languages": None, + "keywords": None, "prompt": None, "response_format": None, "temperature": None, # openai defaults this to 0 "timestamp_granularities": None, + "stream": None, } non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} @@ -3552,6 +3573,9 @@ def get_optional_params_transcription( openai_params=OPENAI_TRANSCRIPTION_PARAMS, additional_drop_params=kwargs.get("additional_drop_params", None), ) + extra_body: Final = optional_params.get("extra_body") + if isinstance(extra_body, dict) and not extra_body: + optional_params.pop("extra_body") return optional_params @@ -6111,6 +6135,7 @@ def _get_model_info_helper( ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None), + cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None @@ -8958,7 +8983,13 @@ class ProviderConfigManager: return XAIAudioTranscriptionConfig() elif litellm.LlmProviders.OPENAI == provider: - if "gpt-4o" in model: + if model == "gpt-transcribe": + from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, + ) + + return OpenAIGPTTranscribeAudioTranscriptionConfig() + elif "gpt-4o" in model: return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d5490f5df9e..cac0aec3830 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,4 +1,264 @@ { + "azure/gpt-live-transcribe": { + "deprecation_date": "2028-02-01", + "input_cost_per_second": 0.000283333333333, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate": { + "deprecation_date": "2027-05-06", + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.0005666666666666667, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_second": 0.0005666666666666667, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096 + }, + "azure/gpt-transcribe": { + "deprecation_date": "2028-02-01", + "input_cost_per_second": 7.5e-05, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_deployment_api": 1, + "transcription_json_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740", + "supported_endpoints": [ + "/v1/audio/transcriptions", + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio", + "text" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_native_streaming": true + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -6079,6 +6339,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-07-31", "input_cost_per_audio_token": 3.2e-05, @@ -6089,6 +6350,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6108,12 +6372,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-07-31", "input_cost_per_audio_token": 1e-05, @@ -6124,6 +6391,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", @@ -6143,6 +6413,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -6322,6 +6594,9 @@ "input_cost_per_second": 0.0002833333333333333, "litellm_provider": "azure", "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -26739,9 +27014,9 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -35718,6 +35993,9 @@ "max_output_tokens": 32000, "max_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -35774,6 +36052,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -35811,6 +36091,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -50035,9 +50317,9 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -60070,6 +60352,9 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_only": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -60082,7 +60367,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -60100,7 +60386,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -60126,7 +60413,13 @@ "max_output_tokens": 2000, "max_tokens": 2000, "mode": "realtime", + "output_cost_per_second": 0.0005666666666666667, "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls" + ], "supported_modalities": [ "audio" ], @@ -60135,7 +60428,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, @@ -70825,6 +71119,9 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -70865,66 +71162,6 @@ "supports_audio_output": true, "supports_function_calling": true }, - "azure/gpt-live-transcribe": { - "deprecation_date": "2028-02-01", - "input_cost_per_second": 0.000283333333333, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "audio_transcription", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/realtime", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true - }, - "azure/gpt-transcribe": { - "deprecation_date": "2028-02-01", - "input_cost_per_second": 7.5e-05, - "litellm_provider": "azure", - "mode": "audio_transcription", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_endpoints": [ - "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" - ], - "supported_modalities": [ - "text", - "audio" - ], - "supported_output_modalities": [ - "text" - ], - "supports_audio_input": true - }, - "azure/gpt-realtime-translate": { - "deprecation_date": "2027-05-06", - "input_cost_per_second": 0.000566666666667, - "litellm_provider": "azure", - "max_input_tokens": 32000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "realtime", - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", - "supported_modalities": [ - "audio" - ], - "supported_output_modalities": [ - "text", - "audio" - ], - "supports_audio_input": true, - "supports_audio_output": true - }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, "litellm_provider": "aihubmix", diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index d09199ed138..a867bc88fbf 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -74,6 +74,10 @@ - {id: llm.bedrock_native.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native invoke stream"} - {id: llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock invoke missing fields and invalid temperature"} - {id: llm.ocr.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: ocr, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.13 / LIT-4778", rationale: "OCR missing document rejected"} +- {id: llm.realtime.openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: translation, streaming: stream, assertions: [works], source: "realtime_endpoints/endpoints.py", rationale: "Dedicated translation client-secret, raw SDP, and WebSocket paths emit translated audio and transcript deltas"} +- {id: llm.realtime.azure_openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: translation, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA translation session emits translated audio and transcript deltas"} +- {id: llm.realtime.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/realtime/handler.py", rationale: "gpt-live-transcribe and gpt-realtime-whisper emit live transcript deltas"} +- {id: llm.realtime.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA live transcription emits transcript deltas"} - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} @@ -91,6 +95,8 @@ - {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"} - {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"} - {id: llm.audio_transcriptions.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.7 / LIT-4778", rationale: "Transcription empty file and missing model are rejected"} +- {id: llm.audio_transcriptions.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "gpt-transcribe streams typed transcript delta and done events"} +- {id: llm.audio_transcriptions.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure gpt-transcribe streams typed transcript delta and done events over the v1 API"} - {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"} - {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"} - {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index d089b9c1ed8..ba2adcb1626 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -86,6 +86,8 @@ LlmCapability = Literal[ "tool_search", "tool_search_history", "tool_use", + "transcription", + "translation", "upstream_stream_failure", "vision", "web_search", @@ -156,14 +158,7 @@ class OtherCell(_Base): Cell = Annotated[ - LlmCell - | MgmtCell - | McpCell - | ReliabilityCell - | QuotaCell - | LoggingCell - | GuardrailCell - | OtherCell, + LlmCell | MgmtCell | McpCell | ReliabilityCell | QuotaCell | LoggingCell | GuardrailCell | OtherCell, Field(discriminator="module"), ] diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index d55316ca429..dad5e326eb4 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -693,6 +693,37 @@ def test_realtime_webrtc_http_routes_classified_as_llm_api(route): assert RouteChecks.is_management_route(route=route) is False +@pytest.mark.parametrize( + "route", + [ + "/realtime/translations", + "/v1/realtime/translations", + "/openai/v1/realtime/translations", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", + ], +) +def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route): + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["openai_routes"], + ) + + assert RouteChecks.is_llm_api_route(route=route) is True + assert RouteChecks.is_management_route(route=route) is False + assert ( + RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=valid_token, + ) + is True + ) + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index de76c7257cf..03cfe3fa67e 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -10,13 +10,21 @@ Pins (PR2): from __future__ import annotations import io +from collections.abc import Callable +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi.testclient import TestClient +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep @pytest.fixture @@ -119,9 +127,7 @@ def patched_transcription(monkeypatch): return data monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - monkeypatch.setattr( - proxy_server, "check_file_size_under_limit", lambda **kwargs: True - ) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", lambda **kwargs: True) async def _form_data(request): from starlette.datastructures import FormData, UploadFile @@ -153,6 +159,47 @@ def patched_transcription_error(monkeypatch, patched_transcription): yield +@pytest.fixture +def patched_transcription_stream(monkeypatch, patched_transcription): + class _FakeEvent: + def model_dump_json(self): + return '{"type":"transcript.text.done","text":"hello world"}' + + class _FakeAsyncStream: + def __init__(self): + self.closed = False + + def __aiter__(self): + async def _events(): + yield _FakeEvent() + + return _events() + + async def aclose(self): + self.closed = True + + async def _form_data(request): + from starlette.datastructures import FormData, UploadFile + + upload = UploadFile( + filename="audio.mp3", + file=io.BytesIO(b"\x00\x01\x02"), + ) + return FormData([("file", upload), ("model", "gpt-transcribe"), ("stream", "true")]) + + stream = _FakeAsyncStream() + + async def _llm_call(): + return stream + + async def _fake_route_request(*args, **kwargs): + return _llm_call() + + monkeypatch.setattr(proxy_server, "get_form_data", _form_data) + monkeypatch.setattr(proxy_server, "route_request", _fake_route_request) + yield stream + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_happy_path(client, auth_as, patched_speech, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (happy).""" @@ -254,3 +301,58 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error, response = client.post(path, files=files, data=data) assert response.status_code == 500 assert len(response.content) > 0 + + +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcription_stream, path): + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + data = {"model": "gpt-transcribe", "stream": "true"} + with auth_as(): + response = client.post(path, files=files, data=data) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\n' + assert patched_transcription_stream.closed is True + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize( + "configuration,expected_status", + [("default", 400), ("model", 400), ("policy", 400), ("pre_call", 200), ("disabled", 200)], +) +def test_streaming_transcription_rejects_applicable_output_guardrails( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + configuration: str, + expected_status: int, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="transcription-output", + event_hook=GuardrailEventHooks.pre_call if configuration == "pre_call" else GuardrailEventHooks.post_call, + default_on=configuration in ("default", "pre_call"), + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + proxy_server.llm_router.get_model_list.return_value = ( + [{"litellm_params": {"guardrails": ["transcription-output"]}}] if configuration == "model" else [] + ) + if configuration == "policy": + pipeline: Final = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="transcription-output")]) + proxy_server.proxy_logging_obj.pre_call_hook.side_effect = lambda **kwargs: { + **kwargs["data"], + "metadata": {"_guardrail_pipelines": [("transcription-policy", pipeline)]}, + } + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == expected_status + if expected_status == 400: + assert "stream=false" in response.json()["error"]["message"] + assert "hello world" not in response.text + else: + assert '"text":"hello world"' in response.text diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 18839a65d62..d79c5669ca9 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -5,11 +5,10 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest - - from fastapi import FastAPI from fastapi.testclient import TestClient +import litellm from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.public_endpoints import router from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets @@ -74,32 +73,20 @@ def test_get_provider_create_fields(): ), "Expected at least one provider to have detailed credential fields" -def test_get_litellm_model_cost_map_returns_cost_map(): - app = FastAPI() +def test_get_litellm_model_cost_map_returns_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + model_cost: Final = { + "audio-model": {"input_cost_per_second": 0.1, "mode": "audio_transcription"}, + "token-model": {"input_cost_per_token": 0.01, "mode": "chat"}, + } + monkeypatch.setattr(litellm, "model_cost", model_cost) + app: Final = FastAPI() app.include_router(router) - client = TestClient(app) + client: Final = TestClient(app) - response = client.get("/public/litellm_model_cost_map") + response: Final = client.get("/public/litellm_model_cost_map") assert response.status_code == 200 - payload = response.json() - assert isinstance(payload, dict) - assert len(payload) > 0, "Expected model cost map to contain at least one model" - - # Verify the structure contains expected keys for at least one model - # Check for a common model like gpt-4 or gpt-3.5-turbo - model_keys = list(payload.keys()) - assert len(model_keys) > 0 - - # Verify at least one model has expected cost fields - sample_model = model_keys[0] - sample_model_data = payload[sample_model] - assert isinstance(sample_model_data, dict) - # Check for common cost fields that should be present - assert ( - "input_cost_per_token" in sample_model_data - or "output_cost_per_token" in sample_model_data - ) + assert response.json() == model_cost def test_public_ai_hub_info_is_public_by_default(monkeypatch): diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index f5c97142dde..f9f75c39db4 100644 --- a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -15,16 +15,25 @@ import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient - -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ConfigGeneralSettings, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) from litellm.proxy.realtime_endpoints.endpoints import ( + _ALLOWED_SESSION_TYPES, + _coerce_realtime_session_type, _decode_realtime_token_payload, _encode_realtime_token_payload, + _prepare_client_secret_session, +) +from litellm.types.realtime import ( + RealtimeAudioInputConfig, + RealtimeAudioTranscriptionConfig, + RealtimeClientSecretRequest, + RealtimeSessionAudioConfig, + RealtimeSessionConfig, ) # --- Unit tests: token encode/decode helpers --- @@ -117,18 +126,107 @@ def proxy_app(monkeypatch): from litellm.proxy import proxy_server monkeypatch.setattr(proxy_server, "master_key", "sk-test-master-key") + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", True) return proxy_server.app +def test_non_billable_realtime_protocols_default_to_disabled(): + assert ConfigGeneralSettings().allow_non_billable_realtime_protocols is False + + +@pytest.mark.parametrize( + ("path", "body"), + ( + ( + "/v1/realtime/client_secrets", + {"model": "gpt-realtime-2"}, + ), + ( + "/v1/realtime/translations/client_secrets", + { + "model": "gpt-realtime-translate", + "session": {"type": "translation", "model": "gpt-realtime-translate"}, + }, + ), + ( + "/v1/realtime/transcription_sessions", + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + ), + ), +) +def test_non_billable_realtime_credential_endpoints_require_opt_in( + proxy_app, + monkeypatch, + path, + body, +): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") + try: + client = TestClient(proxy_app, raise_server_exceptions=False) + with patch( # test-quality-ok: endpoint gate must prove routing is never reached + "litellm.proxy.proxy_server.route_request" + ) as mock_route_request: + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json=body, + ) + + assert response.status_code == 403 + assert "bypasses LiteLLM billing" in response.json()["detail"] + mock_route_request.assert_not_called() + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + ( + "/v1/realtime/calls", + "/v1/realtime/translations/calls", + ), +) +def test_non_billable_realtime_sdp_endpoints_require_opt_in( + proxy_app, + monkeypatch, + path, +): + from litellm.proxy import proxy_server + + monkeypatch.setitem(proxy_server.general_settings, "allow_non_billable_realtime_protocols", False) + token_payload = _encode_realtime_token_payload( + ephemeral_key="epk", + model_id="gpt-realtime-2", + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + ) + encrypted_token = encrypt_value_helper(token_payload) + client = TestClient(proxy_app, raise_server_exceptions=False) + with patch( # test-quality-ok: endpoint gate must prove routing is never reached + "litellm.proxy.proxy_server.route_request" + ) as mock_route_request: + response = client.post( + path, + headers={"Authorization": f"Bearer {encrypted_token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 403 + assert "bypasses LiteLLM billing" in response.json()["detail"] + mock_route_request.assert_not_called() + + @pytest.fixture def mock_route_request_client_secrets(): """Mock route_request to return a fake upstream client_secrets response.""" future_expires_at = int(time.time()) + 3600 mock_resp = MagicMock(spec=httpx.Response) mock_resp.status_code = 200 - mock_resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) + mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode() mock_resp.headers = {} mock_resp.json.return_value = { @@ -215,9 +313,7 @@ async def test_client_secrets_success_with_mock( mock_pre_call_hook, ): """POST /v1/realtime/client_secrets returns 200 with valid auth and mocked upstream.""" - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -275,13 +371,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -312,12 +402,8 @@ async def test_client_secrets_transcription_routes_on_nested_model( async def _inner(): resp = MagicMock(spec=httpx.Response) resp.status_code = 200 - resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) - resp.content = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ).encode() + resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' + resp.content = (f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}').encode() resp.headers = {} resp.json.return_value = { "value": "upstream_ephemeral_key", @@ -351,13 +437,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -367,10 +447,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( session = captured["data"]["session"] assert session["type"] == "transcription" assert "model" not in session - assert ( - session["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert session["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" encrypted_value = response.json()["value"] decoded = _decode_realtime_token_payload( decrypt_value_helper( @@ -530,10 +607,7 @@ async def test_realtime_calls_replays_transcription_session_type( ) assert captured["session"]["type"] == "transcription" - assert ( - captured["session"]["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert captured["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" # --- transcription_sessions endpoint --- @@ -587,8 +661,14 @@ def test_transcription_sessions_requires_auth(proxy_app): @pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + {"model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"input_audio_transcription": {"model": "gpt-4o-realtime-preview"}, "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "gpt-realtime-whisper"}}, +]) async def test_transcription_sessions_rejects_disallowed_resolved_model( - proxy_app, + proxy_app, body, ): proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( user_id="test-user", @@ -605,9 +685,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json=body, ) assert response.status_code == 403 @@ -651,9 +729,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -696,9 +772,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -751,9 +825,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -786,6 +858,20 @@ async def test_realtime_transcription_websocket_default_model_checks_key_scope() assert "is not available for this API key" in close_kwargs["reason"] +def test_realtime_transcription_upstream_query_omits_model(): + from litellm.proxy import proxy_server + + assert ( + proxy_server._resolve_realtime_upstream_query_model( + model="gpt-live-transcribe", + intent="transcription", + is_translation=False, + route_model="gpt-live-transcribe", + ) + is None + ) + + @pytest.mark.asyncio async def test_realtime_transcription_websocket_default_model_checks_team_scope(): from litellm.proxy import proxy_server @@ -947,13 +1033,12 @@ async def test_transcription_sessions_encrypts_client_secret( POST /v1/realtime/transcription_sessions returns 200 and the ephemeral key under client_secret.value must be encrypted (never the raw upstream key). """ - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") captured_route_type = {} async def _capturing_route(*args, **kwargs): captured_route_type["route_type"] = kwargs.get("route_type") + captured_route_type["session"] = kwargs["data"]["transcription_session"] return await mock_route_request_transcription_sessions(*args, **kwargs) try: @@ -976,6 +1061,8 @@ async def test_transcription_sessions_encrypts_client_secret( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, json={ + "model": "gpt-realtime-whisper", + "audio": {"input": {"transcription": {"model": "gpt-live-transcribe", "language": "en"}}}, "input_audio_format": "pcm16", "input_audio_transcription": {"model": "gpt-realtime-whisper"}, }, @@ -993,16 +1080,16 @@ async def test_transcription_sessions_encrypts_client_secret( assert decrypted is not None assert "upstream_ephemeral_key" in decrypted # Routed through the dedicated transcription_sessions route type. - assert ( - captured_route_type["route_type"] - == "acreate_realtime_transcription_session" - ) + assert captured_route_type["route_type"] == "acreate_realtime_transcription_session" + assert captured_route_type["session"]["input_audio_transcription"]["model"] == "gpt-realtime-whisper" + assert captured_route_type["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", "language": "en", + } finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) def test_session_type_coerced_for_unknown_value(): - """An unrecognized session_type in the token falls back to 'realtime'.""" payload = _encode_realtime_token_payload( ephemeral_key="epk", model_id="gpt-4o", @@ -1011,12 +1098,222 @@ def test_session_type_coerced_for_unknown_value(): expires_at=None, session_type="INJECTED_TYPE", ) - # Force-deserialize and check the coercion that happens in proxy_realtime_calls. decoded = json.loads(payload) - session_type = decoded.get("session_type") or "realtime" - if session_type not in ("realtime", "transcription"): - session_type = "realtime" - assert session_type == "realtime" + assert decoded["session_type"] == "INJECTED_TYPE" + assert _coerce_realtime_session_type("INJECTED_TYPE") == "realtime" + assert _coerce_realtime_session_type(None) == "realtime" + for allowed_session_type in _ALLOWED_SESSION_TYPES: + assert _coerce_realtime_session_type(allowed_session_type) == allowed_session_type + + +@pytest.mark.asyncio +async def test_translation_client_secret_rejects_disallowed_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-live-transcribe"), + ) + ), + ), + ) + + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available for this API key"): + await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_binds_authorized_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-realtime-whisper"), + ) + ), + ), + ) + + model, session_data, session_type = await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + assert model == "gpt-realtime-translate" + assert session_type == "translation" + assert session_data is not None + assert session_data["model"] == "gpt-realtime-translate" + assert session_data["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/client_secrets", + "/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/client_secrets", + ], +) +def test_translation_client_secret_aliases_bind_token_family( + proxy_app, + mock_route_request_client_secrets, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-realtime-translate"], + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_client_secrets(*args, **kwargs) + + try: + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-realtime-translate"}, + ) + + assert response.status_code == 200 + assert captured["route_type"] == "acreate_realtime_translation_client_secret" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + decrypted = decrypt_value_helper( + response.json()["value"], + key="client_secret.value", + exception_type="debug", + ) + decoded = _decode_realtime_token_payload(decrypted or "") + assert decoded is not None + assert decoded["session_type"] == "translation" + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/calls", + "/realtime/translations/calls", + "/openai/v1/realtime/translations/calls", + ], +) +def test_translation_calls_aliases_route_translation_session( + proxy_app, + mock_route_request_realtime_calls, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id="gpt-realtime-translate", + user_id="test-user", + team_id=None, + expires_at=int(time.time()) + 3600, + session_type="translation", + ) + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_realtime_calls(*args, **kwargs) + + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 201 + assert captured["route_type"] == "arealtime_translation_calls" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + + +@pytest.mark.parametrize( + "session_type,path", + [ + ("realtime", "/v1/realtime/translations/calls"), + ("translation", "/v1/realtime/calls"), + ], +) +def test_realtime_calls_reject_cross_family_token(proxy_app, session_type, path): + model = "gpt-realtime-translate" if session_type == "translation" else "gpt-realtime-2" + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id=model, + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + session_type=session_type, + ) + ) + response = TestClient(proxy_app).post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + assert response.status_code == 401 + assert response.json()["error"] == "Token is not valid for this Realtime endpoint" @pytest.mark.asyncio @@ -1142,9 +1439,7 @@ async def test_transcription_sessions_returns_upstream_error_verbatim( return _inner() - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -1184,9 +1479,7 @@ async def test_transcription_sessions_wraps_route_exception( async def _raise_http(*args, **kwargs): raise HTTPException(status_code=403, detail="Model not allowed") - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") try: client = TestClient(proxy_app, raise_server_exceptions=False) with ( diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index df05cf0987e..80d621f471d 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -25,6 +25,7 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient +from starlette.datastructures import URL import litellm import litellm.proxy.proxy_server as proxy_server_module @@ -10867,7 +10868,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): def test_realtime_websocket_route_aliases_registered(): - """Realtime sessions reach the proxy via three path aliases stacked on + """Realtime sessions reach the proxy via six path aliases stacked on `realtime_websocket_endpoint`. Dropping any of them silently 405s WebSocket upgrades because the catch-all `/openai/{endpoint:path}` HTTP passthrough only declares HTTP methods. The aliases must also be @@ -10883,7 +10884,14 @@ def test_realtime_websocket_route_aliases_registered(): websocket_paths = {route.path for route in app.routes if isinstance(route, WebSocketRoute)} openai_routes = LiteLLMRoutes.openai_routes.value - for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + for expected in ( + "/openai/v1/realtime", + "/v1/realtime", + "/realtime", + "/openai/v1/realtime/translations", + "/v1/realtime/translations", + "/realtime/translations", + ): assert expected in websocket_paths, ( f"{expected!r} missing from registered WebSocket routes; the " f"realtime endpoint will 405 for clients hitting this path." @@ -10902,7 +10910,7 @@ def _lit6973_fake_realtime_ws() -> MagicMock: ws = MagicMock() ws.headers = {} ws.scope = {"headers": [], "type": "websocket"} - ws.url = "ws://testserver/v1/realtime" + ws.url = URL("ws://testserver/v1/realtime") ws.accept = AsyncMock() ws.send_text = AsyncMock() ws.close = AsyncMock() diff --git a/tests/unit/cookbook/__init__.py b/tests/unit/cookbook/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/cookbook/test_gpt_realtime_translate.py b/tests/unit/cookbook/test_gpt_realtime_translate.py new file mode 100644 index 00000000000..e87005d1bad --- /dev/null +++ b/tests/unit/cookbook/test_gpt_realtime_translate.py @@ -0,0 +1,43 @@ +import asyncio +import base64 +import json +import wave +from pathlib import Path +from types import SimpleNamespace +from typing import Final, cast + +import pytest +from websockets.asyncio.client import ClientConnection + +from cookbook import gpt_realtime_translate as translate + + +@pytest.mark.asyncio +async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(translate, "OUTPUT_IDLE_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(translate, "INITIAL_RESPONSE_TIMEOUT_SECONDS", 0.1) + audio: Final = bytes(480) + events: Final = iter( + ( + {"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}, + {"type": "error", "error": {"message": "session closed"}}, + ) + ) + + async def recv() -> str: + event: Final = next(events) + if event["type"] == "session.output_audio.delta": + await asyncio.sleep(0.03) + return json.dumps(event) + + sender_finished: Final = asyncio.Event() + sender_finished.set() + output: Final = tmp_path / "translation.wav" + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished + ) + + assert result == 'Realtime API error: {"message": "session closed"}' + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(240) == audio diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 0afd989272e..6477ea5e9fe 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -29,6 +29,7 @@ from litellm.llms.gemini.image_generation.cost_calculator import ( from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, @@ -42,6 +43,39 @@ from litellm.types.utils import ( ) +def test_realtime_cached_modality_breakdown_matches_prompt_cost(_local_model_cost_map): + model: Final = "gpt-realtime-2.1-mini" + rates: Final = litellm.model_cost[model] + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=400, + image_tokens=200, + cached_tokens=300, + cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=150, image_tokens=50), + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + breakdown: Final = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) + cached_cost: Final = ( + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + ) + uncached_cost: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + ) + + assert breakdown.cache_read_cost == pytest.approx(cached_cost) + assert prompt_cost == pytest.approx(uncached_cost + cached_cost) + + @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -299,8 +333,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map): ) - - def test_image_tokens_with_custom_pricing(): """Test that image_tokens in completion are properly costed with output_cost_per_image_token.""" from unittest.mock import patch @@ -1950,6 +1982,10 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_tokens_details: PromptTokensDetailsResult = { "cache_hit_tokens": 0, "cache_hit_audio_tokens": 0, + "cached_text_tokens": 0, + "cached_audio_tokens": 0, + "cached_image_tokens": 0, + "has_cached_tokens_details": False, "cache_creation_tokens": 0, "cache_creation_token_details": CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, @@ -2185,10 +2221,6 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo assert round(cost, 10) == round(expected_cost, 10) - - - - def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map): usage = Usage( prompt_tokens=0, @@ -2367,8 +2399,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode assert base == located - - def test_vertex_uplift_invalid_multiplier_defaults_to_one(): """A malformed multiplier in the cost map degrades to base pricing, never raises.""" from litellm.litellm_core_utils.llm_cost_calc.utils import ( @@ -3585,8 +3615,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima assert cost == pytest.approx(0.07) - - @pytest.mark.parametrize( ("custom_llm_provider", "model"), [ diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 7e6d4d24905..94b0d215bf4 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from dataclasses import dataclass from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.realtime_streaming import ( client_sent_openai_beta_realtime_header, ) from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks @@ -602,6 +603,42 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): assert "audio" not in session +@pytest.mark.asyncio +async def test_translation_session_update_omits_session_type(): + client_ws = MagicMock() + client_ws.scope = {"headers": []} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": {"output": {"language": "fr"}}, + }, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + translation_session=True, + ) + + await streaming.client_ack_messages() + + sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) + assert "type" not in sent_to_backend["session"] + assert sent_to_backend["session"]["audio"]["output"]["language"] == "fr" + + def test_translate_event_to_beta_renames_delta_types(): ev = RealTimeStreaming._translate_event_to_beta( {"type": "response.output_audio.delta", "delta": "abc", "event_id": "e1"} @@ -1023,6 +1060,93 @@ async def test_transcription_session_update_enforces_authorized_nested_model(): assert streaming._is_transcription_session is True +@pytest.mark.asyncio +async def test_translation_session_update_rejects_disallowed_nested_transcription_model() -> None: + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + translation_session=True, + ) + + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available"): + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe"}, + } + }, + }, + } + ) + ) + backend_ws.send.assert_not_awaited() + assert streaming._is_transcription_session is False + + +@pytest.mark.asyncio +async def test_translation_session_update_binds_nested_transcription_model() -> None: + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + translation_session=True, + ) + + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-realtime-whisper", "language": "en"}, + } + }, + }, + } + ) + ) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe", "language": "fr"}, + } + }, + }, + } + ) + ) + + first_sent = json.loads(backend_ws.send.await_args_list[0].args[0]) + second_sent = json.loads(backend_ws.send.await_args_list[1].args[0]) + assert first_sent["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + assert second_sent["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", + "language": "fr", + } + assert streaming._is_transcription_session is False + + @pytest.mark.asyncio async def test_normal_realtime_session_keeps_nested_transcription_model(): backend_ws = MagicMock() @@ -2786,6 +2910,248 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] +@pytest.mark.parametrize( + "event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"] +) +def test_translation_audio_duration_is_finalized_once(event_type: str): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + payload = base64.b64encode(bytes(48000)).decode() + streaming._capture_translation_output_audio({"type": event_type, "delta": payload}) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("output_bytes", (0, 48000)) +async def test_translation_disconnect_bills_sent_input_audio( + output_bytes: int, +) -> None: + import base64 + + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), + backend_ws=backend, + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} + ) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + assert streaming.messages == [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": output_bytes / 48000}, + } + ] + + +@pytest.mark.asyncio +async def test_translation_failed_audio_send_is_not_billed() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock(side_effect=RuntimeError("send failed")) + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + + with pytest.raises(RuntimeError, match="send failed"): + await streaming._send_to_backend(json.dumps({"type": "session.input_audio_buffer.append", "audio": "AAAA"})) + streaming._finalize_translation_usage() + + assert streaming.messages == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retain_close", (False, True)) +@pytest.mark.parametrize("reported_input,expected_input", [(None, 2.0), (0.0, 0.0), (0.25, 0.25)]) +async def test_translation_terminal_usage_fills_only_missing_input_duration( + monkeypatch: pytest.MonkeyPatch, retain_close: bool, reported_input: float | None, expected_input: float +) -> None: + import base64 + + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*" if retain_close else None) + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=MagicMock(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) + ) + close_event: Final = { + "type": "session.closed", + "usage": { + "type": "duration", + "output_seconds": 0.5, + **({"input_seconds": reported_input} if reported_input is not None else {}), + }, + } + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert sum(item.get("input_seconds") or 0.0 for item in usage) == expected_input + assert sum(item.get("output_seconds") or 0.0 for item in usage) == 0.5 + + +def test_translation_audio_duration_uses_session_output_format(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"output": {"format": {"type": "audio/pcmu", "rate": 8000}}}}, + } + ) + streaming._capture_translation_output_audio( + { + "type": "session.output_audio.delta", + "delta": base64.b64encode(bytes(8000)).decode(), + } + ) + streaming._finalize_translation_usage() + + assert streaming.messages[-1]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +def test_translation_does_not_duplicate_provider_duration_usage(): + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + streaming.messages.append({"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}}) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + + +def test_translation_prefers_provider_duration_over_audio_byte_estimate(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5} + + +@pytest.mark.parametrize( + ("output_audio_bytes", "expected_usage"), + [ + (0, {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.0}), + (48000, {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0}), + ], +) +def test_translation_preserves_input_only_provider_usage( + output_audio_bytes: int, expected_usage: Mapping[str, str | float] +) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = output_audio_bytes + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == expected_usage + + +def test_translation_retained_input_only_close_event_bills_captured_output(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*") + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + close_event: Final = {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage_events: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert len(usage_events) == 2 + assert sum(usage.get("input_seconds", 0.0) for usage in usage_events) == 0.25 + assert sum(usage.get("output_seconds", 0.0) for usage in usage_events) == 1.0 + + +@pytest.mark.parametrize("malformed_output", [-1.0, float("nan"), float("inf"), 10**1000]) +def test_translation_malformed_provider_duration_uses_captured_audio(malformed_output: float | int) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=MagicMock(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._translation_output_audio_bytes = 48000 + + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": malformed_output}} + ) + streaming._finalize_translation_usage() + + close_events: Final = tuple(event for event in streaming.messages if event.get("type") == "session.closed") + assert len(close_events) == 1 + assert close_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0} + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() diff --git a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py index 73f43ec8d8a..de6deb4de5a 100644 --- a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py @@ -259,6 +259,21 @@ async def test_construct_url_v1_protocol(): assert url.count("/realtime") == 1 +def test_construct_url_translation_protocol(): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + + url = AzureOpenAIRealtime()._construct_url( + api_base="https://my-endpoint.openai.azure.com", + model="translate-deployment", + api_version=None, + realtime_protocol="GA", + query_params={"model": "translate-deployment"}, + realtime_mode="translation", + ) + + assert url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime/translations?model=translate-deployment" + + @pytest.mark.asyncio @pytest.mark.parametrize("protocol", ["ga", "Ga", "gA", "V1", "v1", "GA"]) async def test_construct_url_case_insensitive_protocol(protocol): diff --git a/tests/unit/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py index 4f1906d80be..267ddaf9a5d 100644 --- a/tests/unit/llms/azure/test_audio_transcriptions.py +++ b/tests/unit/llms/azure/test_audio_transcriptions.py @@ -1,6 +1,8 @@ +import io import json from pathlib import Path from typing import Final +from unittest.mock import MagicMock import httpx import pytest @@ -9,6 +11,8 @@ from openai import AzureOpenAI import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.types.utils import TranscriptionResponse AUDIO_FILE: Final = Path(__file__).parents[3] / "gettysburg.wav" WHISPER_COST_PER_SECOND: Final = 0.0001 @@ -39,3 +43,43 @@ def test_azure_transcription_keeps_the_azure_provider(): assert response._hidden_params["custom_llm_provider"] == "azure" assert json.loads(response.model_dump_json())["text"] == "Four score and seven years ago" + + +@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) +@pytest.mark.parametrize( + ("model", "expected_path"), + [ + ("whisper-1", "/openai/v1/audio/transcriptions"), + ("gpt-transcribe", "/openai/deployments/gpt-transcribe/audio/transcriptions"), + ("custom-transcribe-deployment", "/openai/deployments/custom-transcribe-deployment/audio/transcriptions"), + ], +) +def test_azure_transcription_alias_uses_model_route( + monkeypatch: pytest.MonkeyPatch, model: str, expected_path: str, api_version: str +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == expected_path + assert request.url.params.get("api-version") == ( + None if expected_path.startswith("/openai/v1/") else litellm.AZURE_DEFAULT_API_VERSION + ) + return httpx.Response(200, json={"text": "hello"}) + + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with httpx.Client(transport=httpx.MockTransport(send_response)) as http_client: + monkeypatch.setattr(litellm, "client_session", http_client) + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + response: Final = AzureAudioTranscription().audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="test-key", + api_base="https://example.openai.azure.com", + api_version=api_version, + ) + + assert response.text == "hello" diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index caf941ebd19..26f66d4055c 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -474,6 +474,11 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "avector_store_search", "acreate_skill", "acreate_interaction", + "acreate_realtime_client_secret", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_calls", + "arealtime_translation_calls", ] ], ) diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index f7a88b5ba63..3d1c953492b 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,16 +1,32 @@ import json from unittest.mock import AsyncMock, MagicMock, patch -import httpx import pytest - -from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context +from openai import AsyncOpenAI, omit +class DummySDKConnectionManager: + def __init__(self, connection): + self.connection = connection -@pytest.mark.parametrize( - "api_base", ["https://api.openai.com/v1", "https://api.openai.com"] -) + async def __aenter__(self): + return self.connection + + async def __aexit__(self, exc_type, exc, tb): + return None + + +def make_realtime_sdk_client(): + connection = MagicMock() + connection.send_raw = AsyncMock() + connection.recv_bytes = AsyncMock() + connection.close = AsyncMock() + client = MagicMock(spec=AsyncOpenAI) + client.realtime.connect = MagicMock(return_value=DummySDKConnectionManager(connection)) + return client + + +@pytest.mark.parametrize("api_base", ["https://api.openai.com/v1", "https://api.openai.com"]) def test_openai_realtime_handler_url_construction(api_base): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -59,12 +75,8 @@ def test_openai_realtime_handler_model_parameter_inclusion(): api_base = "https://api.openai.com/" # Test with just model parameter - query_params_model_only: RealtimeQueryParams = { - "model": "gpt-4o-mini-realtime-preview" - } - url = handler._construct_url( - api_base=api_base, query_params=query_params_model_only - ) + query_params_model_only: RealtimeQueryParams = {"model": "gpt-4o-mini-realtime-preview"} + url = handler._construct_url(api_base=api_base, query_params=query_params_model_only) # Verify the URL structure assert url.startswith("wss://api.openai.com/v1/realtime?") @@ -75,9 +87,7 @@ def test_openai_realtime_handler_model_parameter_inclusion(): "model": "gpt-4o-mini-realtime-preview", "intent": "chat", } - url_with_extras = handler._construct_url( - api_base=api_base, query_params=query_params_with_extras - ) + url_with_extras = handler._construct_url(api_base=api_base, query_params=query_params_with_extras) # Verify both parameters are included assert url_with_extras.startswith("wss://api.openai.com/v1/realtime?") @@ -91,11 +101,6 @@ def test_openai_realtime_handler_model_parameter_inclusion(): assert expected_pattern in url_with_extras -import asyncio - -import pytest - - @pytest.mark.asyncio async def test_async_realtime_success(): from litellm.llms.openai.realtime.handler import OpenAIRealtime @@ -109,27 +114,10 @@ async def test_async_realtime_success(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -141,6 +129,7 @@ async def test_async_realtime_success(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) mock_realtime_streaming.assert_called_once() @@ -164,28 +153,10 @@ async def test_async_realtime_url_contains_model(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -197,30 +168,48 @@ async def test_async_realtime_url_contains_model(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called with the correct URL - mock_ws_connect.assert_called_once() - called_url = mock_ws_connect.call_args[0][0] - - # Verify the URL contains the model parameter - assert called_url.startswith("wss://api.openai.com/v1/realtime?") - assert f"model={model}" in called_url - - # Verify proper headers were set (GA default: no OpenAI-Beta unless client sent it) - called_kwargs = mock_ws_connect.call_args[1] - assert "additional_headers" in called_kwargs - additional_headers = called_kwargs["additional_headers"] + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] == model + additional_headers = called_kwargs["extra_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert "OpenAI-Beta" not in additional_headers - # Verify SSL is configured (should be an SSLContext or True, not None or False) - assert called_kwargs["ssl"] is not None - assert called_kwargs["ssl"] is not False + assert called_kwargs["max_retries"] == 0 mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() +@pytest.mark.asyncio +async def test_async_realtime_transcription_omits_sdk_model_query(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + websocket = AsyncMock() + logging_obj = MagicMock() + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: + mock_realtime_streaming.return_value.bidirectional_forward = AsyncMock() + + await handler.async_realtime( + model="gpt-live-transcribe", + websocket=websocket, + logging_obj=logging_obj, + api_key="test-key", + query_params={"intent": "transcription"}, + client=sdk_client, + ) + + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] is omit + assert called_kwargs["extra_query"] == {"intent": "transcription"} + + @pytest.mark.asyncio async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it(): """Upstream WS gets OpenAI-Beta: realtime=v1 only when the client WebSocket included it.""" @@ -240,26 +229,10 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() ] } dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -271,11 +244,12 @@ async def test_async_realtime_forwards_openai_beta_header_when_client_sends_it() api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - mock_ws_connect.assert_called_once() - called_kwargs = mock_ws_connect.call_args[1] - additional_headers = called_kwargs["additional_headers"] + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + additional_headers = called_kwargs["extra_headers"] assert additional_headers["Authorization"] == f"Bearer {api_key}" assert additional_headers["OpenAI-Beta"] == "realtime=v1" @@ -300,28 +274,10 @@ async def test_async_realtime_uses_max_size_parameter(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - shared_context = get_shared_realtime_ssl_context() - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -333,20 +289,14 @@ async def test_async_realtime_uses_max_size_parameter(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called with the max_size parameter - mock_ws_connect.assert_called_once() - called_kwargs = mock_ws_connect.call_args[1] - - # Verify max_size is set (default None for unlimited, matching OpenAI's SDK) - assert "max_size" in called_kwargs - assert called_kwargs["max_size"] is None - # Verify SSL is configured (should be an SSLContext or True, not None or False) - assert called_kwargs["ssl"] is not None - assert called_kwargs["ssl"] is not False - # Default should be None (unlimited) to match OpenAI's official agents SDK - # https://github.com/openai/openai-agents-python/blob/cf1b933660e44fd37b4350c41febab8221801409/src/agents/realtime/openai_realtime.py#L235 + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + connection_options = called_kwargs["websocket_connection_options"] + assert connection_options["max_size"] is REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES + assert connection_options["ssl"] is not None mock_realtime_streaming.assert_called_once() mock_streaming_instance.bidirectional_forward.assert_awaited_once() @@ -371,27 +321,10 @@ async def test_async_realtime_ws_url_has_no_ssl(): dummy_websocket = AsyncMock() dummy_logging_obj = MagicMock() - mock_backend_ws = AsyncMock() - - class DummyAsyncContextManager: - def __init__(self, value): - self.value = value - - async def __aenter__(self): - return self.value - - async def __aexit__(self, exc_type, exc, tb): - return None - - with ( - patch( - "websockets.connect", return_value=DummyAsyncContextManager(mock_backend_ws) - ) as mock_ws_connect, - patch( - "litellm.llms.openai.realtime.handler.RealTimeStreaming" - ) as mock_realtime_streaming, - ): - + sdk_client = make_realtime_sdk_client() + with patch( # test-quality-ok: orchestration test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as mock_realtime_streaming: mock_streaming_instance = MagicMock() mock_realtime_streaming.return_value = mock_streaming_instance mock_streaming_instance.bidirectional_forward = AsyncMock() @@ -403,19 +336,13 @@ async def test_async_realtime_ws_url_has_no_ssl(): api_base=api_base, api_key=api_key, query_params=query_params, + client=sdk_client, ) - # Verify websockets.connect was called - mock_ws_connect.assert_called_once() - called_url = mock_ws_connect.call_args[0][0] - called_kwargs = mock_ws_connect.call_args[1] - - # Verify URL was converted from http:// to ws:// - assert called_url.startswith("ws://localhost:8113/v1/realtime?") - assert f"model={model}" in called_url - - # Verify ssl is None for ws:// URLs (the fix for issue #19222) - assert called_kwargs["ssl"] is None + sdk_client.realtime.connect.assert_called_once() + called_kwargs = sdk_client.realtime.connect.call_args.kwargs + assert called_kwargs["model"] == model + assert "ssl" not in called_kwargs["websocket_connection_options"] @pytest.mark.asyncio @@ -465,3 +392,84 @@ async def test_async_realtime_upstream_handshake_refusal_sends_error_event_then_ assert event["error"]["type"] == "server_error" assert "401" in event["error"]["message"] assert closed and closed[0][0] == 1008 + + +def test_translation_url_uses_dedicated_path(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + url = handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + assert url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + + +@pytest.mark.asyncio +async def test_translation_websocket_uses_direct_transport(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + backend = AsyncMock() + + class TranslationConnectionManager: + async def __aenter__(self): + return backend + + async def __aexit__(self, exc_type, exc, tb): + return None + + websocket = MagicMock() + websocket.scope = {"headers": []} + websocket.close = AsyncMock() + logging_obj = MagicMock() + handler = OpenAIRealtime() + expected_url = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert ( + handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + == expected_url + ) + + with ( + patch("websockets.connect", return_value=TranslationConnectionManager()) as connect, + patch( # test-quality-ok: transport test replaces the unbounded streaming loop + "litellm.llms.openai.realtime.handler.RealTimeStreaming" + ) as streaming, + ): + streaming.return_value.bidirectional_forward = AsyncMock() + await handler.async_realtime( + model="gpt-realtime-translate", + websocket=websocket, + logging_obj=logging_obj, + api_base="https://api.openai.com/v1", + api_key="sk-test", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + + connect.assert_called_once() + assert connect.call_args.args[0] == expected_url + assert streaming.call_args.kwargs["translation_session"] is True + + +@pytest.mark.parametrize("sdk_client", [True, False]) +def test_connection_manager_preserves_transport_settings(sdk_client: bool): + import ssl + from litellm.llms.openai.realtime.handler import OpenAIRealtime + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + client = make_realtime_sdk_client() if sdk_client else MagicMock(spec=AsyncHTTPHandler) + ssl_config = ssl.create_default_context() + with patch("websockets.connect") as connect: + OpenAIRealtime()._create_connection_manager( + api_base="https://example.com", api_key="test", model="gpt-realtime-2.1", + query_params={"model": "gpt-realtime-2.1"}, headers={}, timeout=7.0, + realtime_mode="realtime", ssl_config=ssl_config, client=client, url="wss://example.com/v1/realtime", + ) + options = client.realtime.connect.call_args.kwargs["websocket_connection_options"] if sdk_client else connect.call_args.kwargs + assert options["ssl"] is ssl_config + assert options["open_timeout"] == 7.0 diff --git a/tests/unit/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py index 54f206d098d..af64d9c7c16 100644 --- a/tests/unit/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/unit/llms/openai/realtime/test_transcription_sessions.py @@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import litellm from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -21,9 +22,7 @@ from litellm.types.realtime import RealtimeTranscriptionSessionRequest def test_openai_transcription_session_url(): cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -32,9 +31,7 @@ def test_openai_transcription_session_url_strips_trailing_v1(): """A /v1 suffix must not be duplicated in the path.""" cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com/v1", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com/v1", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -46,10 +43,23 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version(): model="whisper-deploy", api_version="2025-04-01-preview", ) - assert ( - url - == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + assert url == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + + +@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_realtime_http_urls(api_version, model: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + cfg = AzureRealtimeHTTPConfig() + base = "https://my.openai.azure.com" + + assert cfg.get_complete_url(base, model, api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, model, api_version) == (f"{base}/openai/v1/realtime/calls") + assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == ( + f"{base}/openai/v1/realtime/transcription_sessions" ) + litellm.get_model_info.cache_clear() def test_request_resolves_model_returns_none_when_both_absent(): @@ -141,6 +151,30 @@ async def test_client_secret_handler_still_targets_client_secrets_url(): assert kwargs["url"] == "https://api.openai.com/v1/realtime/client_secrets" +@pytest.mark.asyncio +async def test_azure_client_secret_prefers_provider_qualified_routing_model(): + import litellm + + mock_response = MagicMock(spec=httpx.Response) + mock_client = MagicMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=mock_response) + + result = await litellm.acreate_realtime_client_secret( + model="azure/gpt-realtime-2.1", + session={"type": "realtime", "model": "gpt-realtime-2.1"}, + api_base="https://my.openai.azure.com", + api_key="azure-test-key", + api_version="v1", + client=mock_client, + ) + + assert result is mock_response + request = mock_client.post.call_args.kwargs + assert request["url"] == "https://my.openai.azure.com/openai/v1/realtime/client_secrets" + assert request["headers"]["api-key"] == "azure-test-key" + assert request["json"]["session"]["model"] == "gpt-realtime-2.1" + + @pytest.mark.asyncio async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): """ @@ -169,18 +203,14 @@ async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): assert kwargs["url"].endswith("/v1/realtime/transcription_sessions") # The litellm-only routing hint must not be forwarded upstream. assert "model" not in kwargs["json"] - assert kwargs["json"]["input_audio_transcription"] == { - "model": "gpt-realtime-whisper" - } + assert kwargs["json"]["input_audio_transcription"] == {"model": "gpt-realtime-whisper"} def test_append_query_params_skips_existing_keys(): from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler url = "wss://example.com/v1/realtime?model=gpt-4o" - result = BaseLLMHTTPHandler._append_query_params( - url, {"model": "ignored", "intent": "transcription"} - ) + result = BaseLLMHTTPHandler._append_query_params(url, {"model": "ignored", "intent": "transcription"}) assert "model=ignored" not in result assert "intent=transcription" in result diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py new file mode 100644 index 00000000000..06af49f26ca --- /dev/null +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -0,0 +1,281 @@ +import gzip +import json +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncOpenAI + +from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig +from litellm.types.realtime import RealtimeSessionConfig +from litellm.types.router import GenericLiteLLMParams + + +def test_azure_realtime_http_headers_use_ad_token_without_api_key(): + config = AzureRealtimeHTTPConfig() + params = GenericLiteLLMParams(azure_ad_token="test-ad-token") + + assert config.get_extra_headers(params, "", {"X-Trace": "trace"}) == { + "X-Trace": "trace", + "Authorization": "Bearer test-ad-token", + } + assert config.get_extra_headers(params, "test-api-key", {"X-Trace": "trace"}) == {"X-Trace": "trace"} + + +def test_realtime_session_config_supports_translation_and_live_transcription_fields(): + session = RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio={ + "input": { + "transcription": { + "model": "gpt-live-transcribe", + "delay": "minimal", + "languages": ["en", "fr"], + "keywords": ["LiteLLM"], + } + }, + "output": {"language": "es"}, + }, + ) + + assert session.audio is not None + assert session.audio.input is not None + assert session.audio.input.transcription is not None + assert session.audio.input.transcription.delay == "minimal" + assert session.audio.input.transcription.languages == ["en", "fr"] + assert session.audio.input.transcription.keywords == ["LiteLLM"] + assert session.audio.output is not None + assert session.audio.output.language == "es" + + +@pytest.mark.parametrize( + "api_base,expected", + [ + ( + "https://api.openai.com", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ( + "https://api.openai.com/v1", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ], +) +def test_openai_translation_client_secret_url(api_base: str, expected: str): + config = OpenAIRealtimeHTTPConfig() + assert config.get_translation_client_secret_url(api_base, "gpt-realtime-translate") == expected + + +def test_openai_translation_calls_url(): + config = OpenAIRealtimeHTTPConfig() + assert ( + config.get_translation_calls_url("https://api.openai.com/v1", "gpt-realtime-translate") + == "https://api.openai.com/v1/realtime/translations/calls" + ) + + +def test_azure_translation_urls_use_ga_paths(): + config = AzureRealtimeHTTPConfig() + assert ( + config.get_translation_client_secret_url("https://example.openai.azure.com", "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets" + ) + assert ( + config.get_translation_calls_url("https://example.openai.azure.com", "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/calls" + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_custom_translation_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/client_secrets"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + request_data = {"session": {"type": "translation", "model": "gpt-realtime-translate"}} + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://api.openai.com", + api_key="sk-test", + request_data=request_data, + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + ) + + assert response.status_code == 200 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/client_secrets" + assert call["json"] == request_data + + +@pytest.mark.asyncio +async def test_azure_translation_client_secret_supports_entra_bearer_auth(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request( + "POST", + "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets", + ), + ) + ) + handler = BaseLLMHTTPHandler() + + await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.openai.azure.com", + api_key="", + request_data={"session": {"type": "translation", "model": "translate-deployment"}}, + logging_obj=MagicMock(), + timeout=10, + provider_config=AzureRealtimeHTTPConfig(), + model="translate-deployment", + extra_headers={"Authorization": "Bearer entra-token"}, + client=client, + ) + + headers = client.post.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer entra-token" + assert "api-key" not in headers + + +@pytest.mark.asyncio +async def test_translation_calls_use_translation_session_and_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 201, + content=b"v=0\r\n", + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/calls"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://api.openai.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + translation=True, + ) + + assert response.status_code == 201 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/calls" + assert call["headers"]["Content-Type"] == "application/sdp" + assert call["content"] == "v=0\r\n" + + +@pytest.mark.asyncio +async def test_standard_client_secret_uses_openai_sdk_resource(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/client_secrets" + return httpx.Response( + 200, + content=gzip.compress(b'{"value":"ek_test"}'), + headers={"content-encoding": "gzip"}, + ) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + logging_obj = MagicMock() + + response = await handler.async_realtime_client_secret_handler( + api_base="https://example.com", + api_key="sk-test", + request_data={"session": {"type": "realtime", "model": "gpt-realtime-2.1"}}, + logging_obj=logging_obj, + timeout=10, + client=openai_client, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 200 + assert response.json() == {"value": "ek_test"} + assert "content-encoding" not in response.headers + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_openai_sdk_custom_post(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/translations/client_secrets" + assert json.loads(request.content)["session"]["audio"]["output"]["language"] == "es" + return httpx.Response(200, content=gzip.compress(b'{"value":"ek_translation"}'), headers={"content-encoding": "gzip"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.com", + api_key="sk-test", + request_data={ + "session": { + "model": "gpt-realtime-translate", + "audio": {"output": {"language": "es"}}, + } + }, + logging_obj=MagicMock(), + timeout=10, + client=openai_client, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 200 + assert response.json() == {"value": "ek_translation"} + assert "content-encoding" not in response.headers + assert int(response.headers["content-length"]) == len(response.content) + + +@pytest.mark.asyncio +async def test_translation_calls_use_openai_sdk_custom_post(): + async def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1/realtime/translations/calls" + body = await request.aread() + assert request.headers["content-type"] == "application/sdp" + assert body == b"v=0\r\n" + return httpx.Response(201, content=gzip.compress(b"v=0\r\n"), headers={"content-encoding": "gzip"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="ek_test", base_url="https://example.com/v1", http_client=http_client) + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://example.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=MagicMock(), + timeout=10, + model="gpt-realtime-translate", + client=openai_client, + translation=True, + use_openai_sdk=True, + ) + await openai_client.close() + + assert response.status_code == 201 + assert response.text == "v=0\r\n" + assert "content-encoding" not in response.headers + assert int(response.headers["content-length"]) == len(response.content) diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py new file mode 100644 index 00000000000..535b52a17ee --- /dev/null +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -0,0 +1,363 @@ +import io +import json +import wave +from collections.abc import Iterator +from datetime import datetime +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI + +import litellm +from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, +) +from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription +from litellm.types.utils import TranscriptionResponse +from litellm.utils import get_optional_params_transcription + + +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def test_gpt_transcribe_config_uses_native_parameters_and_json(): + config = OpenAIGPTTranscribeAudioTranscriptionConfig() + supported = config.get_supported_openai_params("gpt-transcribe") + assert supported == ["prompt", "response_format", "keywords", "languages", "stream"] + + audio_file = io.BytesIO(b"audio") + request = config.transform_audio_transcription_request( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True}, + litellm_params={}, + ) + assert request.data["response_format"] == "json" + assert request.data["keywords"] == ["LiteLLM"] + assert request.data["languages"] == ["en", "fr"] + assert request.data["stream"] is True + + +def test_gpt_transcribe_optional_params_are_preserved(): + params = get_optional_params_transcription( + model="gpt-transcribe", + custom_llm_provider="openai", + keywords=["LiteLLM", "Realtime API"], + languages=["en", "fr"], + stream=True, + ) + assert params == { + "keywords": ["LiteLLM", "Realtime API"], + "languages": ["en", "fr"], + "stream": True, + } + + +def test_transcription_response_preserves_empty_languages(): + response = TranscriptionResponse(text="hello", languages=[]) + assert response.model_dump()["languages"] == [] + + +def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + 'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + 'data: {"type":"transcript.text.done","text":"hello world",' + '"usage":{"type":"duration","seconds":2.5}}\n\n' + ), + ) + + logging_obj: Final = MagicMock() + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + stream: Final = client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + wrapped: Final = wrap_transcription_stream(stream, logging_obj, datetime(2026, 1, 1)) + received: Final = tuple(wrapped) + wrapped.close() + + assert tuple(event.type for event in received) == ("transcript.text.delta", "transcript.text.done") + logging_obj.success_handler.assert_called_once() + logged_response: Final = logging_obj.success_handler.call_args.args[0] + assert logged_response.text == "hello world" + assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5} + logging_obj.failure_handler.assert_not_called() + + +@pytest.mark.asyncio +async def test_openai_handler_returns_native_typed_stream(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + assert b'name="stream"' in body + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "languages": [], + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + handler = OpenAIAudioTranscription() + result = handler.audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"], "stream": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + logging_obj=logging_obj, + api_key="sk-test", + api_base="https://example.com/v1", + client=openai_client, + atranscription=True, + provider_config=OpenAIGPTTranscribeAudioTranscriptionConfig(), + ) + stream = await result + assert isinstance(stream, AsyncStream) + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now()) + received = [event async for event in wrapped_stream] + await wrapped_stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + assert received[-1].languages == [] + logging_obj.async_success_handler.assert_awaited_once() + logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + assert logged_response.text == "hello world" + assert logged_response.languages == [] + + +@pytest.mark.asyncio +async def test_closed_transcription_stream_without_usage_or_duration_does_not_log_success(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello"}, + {"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 1}}, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + stream = await client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + logging_obj = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now()) + + async for event in wrapped_stream: + assert event.type == "transcript.text.delta" + break + await wrapped_stream.close() + await client.close() + + logging_obj.handle_sync_success_callbacks_for_async_calls.assert_not_called() + logging_obj.async_success_handler.assert_not_awaited() + logging_obj.async_failure_handler.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_atranscription_stream_preserves_duration_for_callback_cost(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-transcribe", messages=[], stream=True, call_type="atranscription", + start_time=datetime.now(), litellm_call_id="transcription-cost-test", function_id="transcription-cost-test", + ) + audio_file = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + + stream = await litellm.atranscription( + model="openai/gpt-transcribe", + file=audio_file, + stream=True, + client=openai_client, + litellm_logging_obj=logging_obj, + ) + received = [event async for event in stream] + await stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + logged_response = logging_obj.model_call_details["async_complete_streaming_response"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.000075) + assert logging_obj.model_call_details["standard_logging_object"]["response_cost"] == pytest.approx(0.000075) + assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) + + +def test_gpt_transcribe_rejects_conflicting_language_inputs(): + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="cannot be used together"): + litellm.transcription( + model="gpt-transcribe", + file=audio_file, + language="en", + languages=["fr"], + api_key="sk-test", + ) + + +@pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"]) +def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None: + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format='json'"): + litellm.transcription( + model=model, + file=audio_file, + response_format="verbose_json", + api_key="sk-test", + ) + + +def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: None) -> None: + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="Realtime API"): + litellm.transcription( + model="gpt-live-transcribe", + file=audio_file, + api_key="sk-test", + ) + + +def test_azure_gpt_transcribe_uses_deployment_scoped_route(): + def send_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ( + "https://example.openai.azure.com/openai/deployments/gpt-transcribe/audio/transcriptions" + f"?api-version={litellm.AZURE_DEFAULT_API_VERSION}" + ) + return httpx.Response( + 200, + json={"text": "hello", "languages": [{"code": "en"}], "usage": {"type": "duration", "seconds": 1}}, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(send_response)) + client = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert response.languages is not None + assert [language.code for language in response.languages] == ["en"] + client.close() + + +@pytest.mark.asyncio +async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + return httpx.Response(200, json={"text": "hello"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncAzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version="2025-04-01-preview", + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = await AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"]}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version="2025-04-01-preview", + client=client, + atranscription=True, + ) + + assert response.text == "hello" + await client.close() diff --git a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py index 703fa13cbc9..a15ba6d7543 100644 --- a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py +++ b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py @@ -9,6 +9,9 @@ TranscriptionVerbose/Diarized type. from unittest.mock import patch +import pytest + +import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, @@ -55,10 +58,7 @@ class TestDiarizedJsonUsageParsing: assert result.usage.seconds == 295.8 def test_usage_duration_object_accepts_float_seconds(self): - assert ( - TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds - == 295.8 - ) + assert TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds == 295.8 class TestTranscriptionDurationNotInResponseBody: @@ -126,6 +126,60 @@ class TestTranscriptionDurationNotInResponseBody: class TestCostCalculatorReadsDurationFromHiddenParams: """The cost calculator should read duration from _hidden_params via completion_cost().""" + @patch( # test-quality-ok: isolates duration selection from the model pricing table + "litellm.cost_calculator.openai_cost_per_second" + ) + def test_completion_cost_prefers_provider_usage_duration(self, mock_cost_fn): + mock_cost_fn.return_value = (0.001, 0.0) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = { + "audio_transcription_duration": 17.5, + "custom_llm_provider": "openai", + } + + completion_cost( + completion_response=response, + model="gpt-transcribe", + call_type="atranscription", + ) + + mock_cost_fn.assert_called_once() + _, kwargs = mock_cost_fn.call_args + assert kwargs["duration"] == 60.0 + + @pytest.mark.parametrize( + ("model", "provider"), + [ + ("gpt-transcribe", "openai"), + ("azure/gpt-transcribe", "azure"), + ], + ) + def test_gpt_transcribe_provider_usage_duration_costs_without_local_decode( + self, + monkeypatch, + model: str, + provider: str, + ): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = {"custom_llm_provider": provider} + + actual = completion_cost( + completion_response=response, + model=model, + call_type="atranscription", + custom_llm_provider=provider, + ) + + assert actual == pytest.approx(0.0045) + @patch("litellm.cost_calculator.openai_cost_per_second") def test_completion_cost_uses_hidden_params_duration(self, mock_cost_fn): """ diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 5d3276dfae1..9b0ac1e531d 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -463,6 +463,30 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map, model: str) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + assert ( + azure_realtime_protocol_for_client( + None, + model=model, + realtime_mode="realtime", + query_params=None, + websocket=_BETA_CLIENT, + ) + == "GA" + ) + with pytest.raises(ValueError, match="requires the Azure OpenAI v1 Realtime API"): + azure_realtime_protocol_for_client( + "beta", + model=model, + realtime_mode="realtime", + query_params=None, + websocket=_BETA_CLIENT, + ) + + async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: connect: Final = _ConnectThatStopsAfterCapturingTheUrl() with patch("websockets.connect", connect): diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index e7cf09909fe..1d884e39682 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -538,6 +538,8 @@ def test_every_openai_sdk_response_error_code_has_explicit_status_mapping(): ("insufficient_quota", 429), ("vector_store_timeout", 504), ("invalid_prompt", 400), + ("data_residency_mismatch", 400), + ("bio_policy", 400), ("invalid_image", 400), ("invalid_image_format", 400), ("invalid_base64_image", 400), diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 62ef9f11c2e..bfb89f98347 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -160,10 +160,6 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 - - - - def test_realtime_stream_combines_text_and_audio_token_details(): """Realtime response.done usage with input_token_details / output_token_details.""" from litellm.cost_calculator import RealtimeAPITokenUsageProcessor @@ -575,6 +571,15 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type(): assert _transcription_usage_cost({}, {}) == 0.0 +@pytest.mark.parametrize("malformed_seconds", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_transcription_ignores_invalid_provider_duration(malformed_seconds: float | int | bool) -> None: + from litellm.cost_calculator import _get_transcription_usage_duration + + response: Final = SimpleNamespace(usage={"type": "duration", "seconds": malformed_seconds}) + + assert _get_transcription_usage_duration(response) is None + + def test_get_transcription_model_falls_back_to_session_model(monkeypatch): """session.model is used when transcription-specific model fields are absent.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -824,7 +829,7 @@ def test_completion_cost_image_generation_reads_deployment_model_info_price_from assert cost == pytest.approx(0.08) -def test_completion_cost_image_generation_registered_deployment_price_keeps_map_token_rates( +def test_completion_cost_image_generation_registered_deployment_applies_custom_image_rate( _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: deployment_id: Final = "gemini-image-deployment-priced-per-image" @@ -1278,8 +1283,6 @@ def test_bedrock_cost_calculator_comparison_with_without_cache(): print(f"Cost with cache: {cost_with_cache}") - - def test_gemini_25_explicit_caching_cost_direct_usage(): """ Test that Gemini 2.5 models correctly calculate costs with explicit caching. @@ -1848,8 +1851,6 @@ def test_cost_margin_with_discount(monkeypatch): print(f" - Expected: ${expected_cost:.6f}") - - def test_completion_cost_extracts_service_tier_from_response(_local_model_cost_map): """Test that completion_cost extracts service_tier from completion_response object.""" from litellm import completion_cost @@ -2600,8 +2601,6 @@ def test_gemini_without_cache_tokens_details(): print("✅ Gemini without cacheTokensDetails works correctly") - - def test_additional_costs_only_for_azure_ai(_local_model_cost_map): """ Test that _get_additional_costs is only called for azure_ai provider. @@ -4921,6 +4920,179 @@ def test_gemini_live_native_audio_limits_and_capabilities_match_vendor_model_car assert info["supports_pdf_input"] is False +@pytest.mark.parametrize("provider", ("openai", "azure")) +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str): + model_name: Final = f"azure/{model}" if provider == "azure" else model + rates: Final = litellm.model_cost[model_name] + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.created", "session": {"model": model}}, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 300, + "total_tokens": 1300, + "input_token_details": { + "text_tokens": 400, + "audio_tokens": 400, + "image_tokens": 200, + "cached_tokens": 300, + "cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50}, + }, + "output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100}, + } + }, + }, + ] + combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events) + actual: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=combined, + custom_llm_provider=provider, + litellm_model_name=model_name, + ) + expected: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + + 200 * rates["output_cost_per_token"] + + 100 * rates["output_cost_per_audio_token"] + ) + + assert actual == pytest.approx(expected) + + +@pytest.mark.parametrize("input_override,output_override", [(None, None), (0.25, 0.75), (0.0, 0.0), (0.0, None)]) +def test_realtime_translation_uses_deployment_rates_before_base_rates( + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + input_override: float | None, + output_override: float | None, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "translation-base", + { + "litellm_provider": "azure", + "mode": "realtime", + "input_cost_per_second": 0.5, + "output_cost_per_second": 1.0, + }, + ) + monkeypatch.setitem( + litellm.model_cost, + "translation-deployment", + { + "litellm_provider": "azure", + "mode": "realtime", + **{ + key: rate + for key, rate in (("input_cost_per_second", input_override), ("output_cost_per_second", output_override)) + if rate is not None + }, + }, + ) + litellm.get_model_info.cache_clear() + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 3.0, "output_seconds": 2.0}} + ] + cost: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=Usage(), + custom_llm_provider="azure", + litellm_model_name="unmapped-provider-deployment", + custom_pricing_model="translation-deployment", + base_pricing_model="translation-base", + ) + + assert cost == pytest.approx( + 3 * (0.5 if input_override is None else input_override) + + 2 * (1.0 if output_override is None else output_override) + ) + + +def test_realtime_translation_duration_cost(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"]) + + +@pytest.mark.parametrize("output_seconds", [None, 2.0]) +def test_realtime_translation_duration_cost_includes_provider_input_usage( + _local_model_cost_map, output_seconds: float | None +): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": { + "type": "duration", + "input_seconds": 3.0, + **({"output_seconds": output_seconds} if output_seconds is not None else {}), + }, + } + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + expected: Final = ( + 3 * litellm.model_cost[model]["input_cost_per_second"] + + (output_seconds or 0) * litellm.model_cost[model]["output_cost_per_second"] + ) + assert actual == pytest.approx(expected) + + +@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( + _local_model_cost_map, malformed_duration: float | int | bool +) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + input_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5}, + } + ] + output_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration}, + } + ] + + input_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + output_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + + assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"]) + assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) + + def test_realtime_honours_deployment_custom_pricing(monkeypatch: pytest.MonkeyPatch) -> None: """Regression: a deployment's pricing override never reached realtime costing. diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index a05c345b5cd..3f9bcb87fc2 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -533,3 +533,36 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): "unknown_root-new_family_models", "vertex_ai-new_family_models", ] + + +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["cache_read_input_image_token_cost"] > 0 + assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"] + assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"] + + +@pytest.mark.parametrize( + "model,mode", + ( + ("gpt-realtime-translate", "realtime"), + ("gpt-live-transcribe", "audio_transcription"), + ("gpt-transcribe", "audio_transcription"), + ), +) +def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["mode"] == azure["mode"] == mode + assert azure["supports_audio_input"] is True + assert azure["supported_endpoints"] + + +def test_model_prices_backup_is_synchronized(prices: dict): + backup: Final = json.loads(BACKUP_PRICES_PATH.read_text()) + + assert backup == prices diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 2c612aa350c..800ce129d97 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -998,6 +998,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/images/generations", "/v1/realtime", "/v1/realtime/transcription_sessions", + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls", "/v1/images/variations", "/v1/images/edits", "/v1/batch", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4e64a98dd65..8ce114ad47f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10227,7 +10227,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_3"]; + get: operations["websocket_realtime_websocket_endpoint_get_6"]; put?: never; post?: never; delete?: never; @@ -10294,6 +10294,60 @@ export interface paths { patch?: never; trace?: never; }; + "/openai/v1/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get_3"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/openai/v1/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_openai_v1_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/openai/v1/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/openai/v1/responses": { parameters: { query?: never; @@ -13109,7 +13163,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get"]; + get: operations["websocket_realtime_websocket_endpoint_get_4"]; put?: never; post?: never; delete?: never; @@ -13176,6 +13230,60 @@ export interface paths { patch?: never; trace?: never; }; + "/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/register": { parameters: { query?: never; @@ -20264,7 +20372,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_2"]; + get: operations["websocket_realtime_websocket_endpoint_get_5"]; put?: never; post?: never; delete?: never; @@ -20331,6 +20439,60 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Proxy Realtime Calls */ + post: operations["proxy_realtime_calls_v1_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Create Realtime Client Secret */ + post: operations["create_realtime_client_secret_v1_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/rerank": { parameters: { query?: never; @@ -26667,7 +26829,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -27977,6 +28139,12 @@ export interface components { * @description opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine */ allow_cli_sso_verification_uri_complete?: boolean | null; + /** + * Allow Non Billable Realtime Protocols + * @description Allow Realtime WebRTC setup endpoints whose inference usage bypasses LiteLLM spend tracking and budget enforcement + * @default false + */ + allow_non_billable_realtime_protocols: boolean; /** * Allow Unmanaged Response Ids * @description If True, lets keys address Responses API ids that this proxy did not issue (raw provider ids, or ids issued before response-id encryption was configured). Such an id carries no owner, so no ownership check can run on it; ids this proxy did issue keep full ownership enforcement. Off by default, in which case an unrecognized response id is rejected with 403 @@ -60743,7 +60911,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_3: { + websocket_realtime_websocket_endpoint_get_6: { parameters: { query?: never; header?: never; @@ -60821,6 +60989,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get_3: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_openai_v1_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_openai_v1_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; responses_api_openai_v1_responses_post: { parameters: { query?: never; @@ -63891,7 +64117,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get: { + websocket_realtime_websocket_endpoint_get_4: { parameters: { query?: never; header?: never; @@ -63969,6 +64195,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; register_client_register_post: { parameters: { query?: { @@ -73215,7 +73499,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_2: { + websocket_realtime_websocket_endpoint_get_5: { parameters: { query?: never; header?: never; @@ -73293,6 +73577,64 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_v1_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + create_realtime_client_secret_v1_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; rerank_v1_rerank_post: { parameters: { query?: never;