diff --git a/litellm/llms/azure/realtime/transcription_sessions.py b/litellm/llms/azure/realtime/transcription_sessions.py new file mode 100644 index 00000000000..da56530a416 --- /dev/null +++ b/litellm/llms/azure/realtime/transcription_sessions.py @@ -0,0 +1,126 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +from pydantic import JsonValue, TypeAdapter + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_AUDIO_FORMATS: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "pcm16": {"type": "audio/pcm", "rate": 24000}, + "g711_ulaw": {"type": "audio/pcmu"}, + "g711_alaw": {"type": "audio/pcma"}, + } +) +_LEGACY_AUDIO_FORMATS: Final[Mapping[str, str]] = MappingProxyType( + {"audio/pcm": "pcm16", "audio/pcmu": "g711_ulaw", "audio/pcma": "g711_alaw"} +) + + +def azure_transcription_client_secret_request( + transcription_session: Mapping[str, JsonValue], model: str +) -> Mapping[str, JsonValue]: + raw_secret: Final = transcription_session.get("client_secret") + expires_after: Final = transcription_session.get( + "expires_after", raw_secret.get("expires_at") if isinstance(raw_secret, dict) else raw_secret + ) + raw_audio: Final = transcription_session.get("audio") + audio: Final = raw_audio if isinstance(raw_audio, dict) else {} + raw_audio_input: Final = audio.get("input") + existing_audio_input: Final = raw_audio_input if isinstance(raw_audio_input, dict) else {} + raw_audio_transcription: Final = existing_audio_input.get("transcription") + audio_transcription: Final = raw_audio_transcription if isinstance(raw_audio_transcription, dict) else {} + raw_transcription: Final = transcription_session.get("input_audio_transcription") + transcription: Final = raw_transcription if isinstance(raw_transcription, dict) else {} + raw_format: Final = transcription_session.get("input_audio_format") + audio_input: Final[JsonValue] = { + **existing_audio_input, + **( + { + "format": _AUDIO_FORMATS[raw_format] + if isinstance(raw_format, str) and raw_format in _AUDIO_FORMATS + else raw_format + } + if "input_audio_format" in transcription_session + else {} + ), + "transcription": {**audio_transcription, **transcription, "model": model}, + **( + {"noise_reduction": transcription_session["input_audio_noise_reduction"]} + if "input_audio_noise_reduction" in transcription_session + else {} + ), + **( + {"turn_detection": transcription_session["turn_detection"]} + if "turn_detection" in transcription_session + else {} + ), + } + session: Final[JsonValue] = { + **{ + key: value + for key, value in transcription_session.items() + if key + not in ( + "model", + "audio", + "client_secret", + "expires_after", + "modalities", + "input_audio_format", + "input_audio_transcription", + "input_audio_noise_reduction", + "turn_detection", + ) + }, + "type": "transcription", + "audio": { + **{key: value for key, value in audio.items() if key != "input"}, + "input": audio_input, + }, + } + return MappingProxyType( + {"session": session, **({"expires_after": expires_after} if expires_after is not None else {})} + ) + + +def azure_transcription_client_secret_response(response: httpx.Response) -> httpx.Response: + if response.is_error: + return response + payload: Final = _JSON.validate_python(response.json()) + if not isinstance(payload, dict): + return response + raw_session: Final = payload.get("session") + session: Final = raw_session if isinstance(raw_session, dict) else {} + raw_audio: Final = session.get("audio") + audio: Final = raw_audio if isinstance(raw_audio, dict) else {} + raw_input: Final = audio.get("input") + audio_input: Final = raw_input if isinstance(raw_input, dict) else {} + raw_format: Final = audio_input.get("format") + format_type: Final = raw_format.get("type") if isinstance(raw_format, dict) else None + input_audio_format: Final = ( + _LEGACY_AUDIO_FORMATS.get(format_type, format_type) if isinstance(format_type, str) else raw_format + ) + client_secret: Final = { + "value": payload.get("value"), + **({"expires_at": payload["expires_at"]} if "expires_at" in payload else {}), + } + response_body: Final = { + **{key: value for key, value in session.items() if key not in ("type", "audio", "expires_at")}, + **({"input_audio_format": input_audio_format} if input_audio_format is not None else {}), + **({"input_audio_transcription": audio_input["transcription"]} if "transcription" in audio_input else {}), + **({"input_audio_noise_reduction": audio_input["noise_reduction"]} if "noise_reduction" in audio_input else {}), + **({"turn_detection": audio_input["turn_detection"]} if "turn_detection" in audio_input else {}), + "client_secret": client_secret, + } + return httpx.Response( + response.status_code, + headers={ + key: value + for key, value in response.headers.items() + if key.lower() not in ("content-length", "content-encoding", "transfer-encoding") + }, + json=response_body, + request=response.request, + ) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 7b9068ff95e..5711e8af749 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -40,6 +40,10 @@ from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ..litellm_core_utils.realtime_streaming import client_websocket_scope from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client +from ..llms.azure.realtime.transcription_sessions import ( + azure_transcription_client_secret_request, + azure_transcription_client_secret_response, +) from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed from ..llms.custom_httpx.http_handler import realtime_ssl_for_url from ..llms.openai.live.handler import OpenAILiveSessions @@ -267,19 +271,27 @@ async def acreate_realtime_transcription_session( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) + is_azure: Final = custom_llm_provider == LlmProviders.AZURE.value + request_data: Final = req.model_dump(exclude_none=not is_azure, exclude_unset=is_azure, exclude={"model"}) # Ensure the upstream body's input_audio_transcription.model matches the # authorized routing model. This prevents a caller from supplying an allowed # top-level model for auth while sneaking a different model into the nested # transcription config that gets forwarded to the provider. if isinstance(request_data.get("input_audio_transcription"), dict): request_data["input_audio_transcription"]["model"] = model_name - return await base_llm_http_handler.async_realtime_transcription_session_handler( + create_session: Final = ( + base_llm_http_handler.async_realtime_client_secret_handler + if is_azure + else base_llm_http_handler.async_realtime_transcription_session_handler + ) + response: Final = await create_session( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params ), - request_data=request_data, + request_data=dict(azure_transcription_client_secret_request(request_data, model_name)) + if is_azure + else request_data, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, provider_config=provider_config, @@ -288,6 +300,7 @@ async def acreate_realtime_transcription_session( client=kwargs.get("client"), api_version=litellm_params.api_version, ) + return azure_transcription_client_secret_response(response) if is_azure else response @wrapper_client diff --git a/tests/unit/llms/azure/realtime/test_http_transformation.py b/tests/unit/llms/azure/realtime/test_http_transformation.py index 714ce9558cf..26080ce0f1b 100644 --- a/tests/unit/llms/azure/realtime/test_http_transformation.py +++ b/tests/unit/llms/azure/realtime/test_http_transformation.py @@ -1,22 +1,35 @@ -from typing import Final +import gzip +import json +from typing import Final, Literal import httpx import pytest import litellm +from litellm.llms.azure.realtime.transcription_sessions import azure_transcription_client_secret_response from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler _DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview" -_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai") +_API_BASES: Final = ( + "https://my.openai.azure.com", + "https://my.openai.azure.com/", + "https://my.openai.azure.com/openai", +) +_AUDIO_FORMATS: Final = ( + ("pcm16", {"type": "audio/pcm", "rate": 24000}), + ("g711_ulaw", {"type": "audio/pcmu"}), + ("g711_alaw", {"type": "audio/pcma"}), +) class _AzureThatRecordsTheRequest: - def __init__(self) -> None: + def __init__(self, reply: dict[str, object] | None = None) -> None: self.requests: Final[list[httpx.Request]] = [] + self.reply: Final = reply or {"value": "ek_upstream"} def __call__(self, request: httpx.Request) -> httpx.Response: self.requests.append(request) - return httpx.Response(200, json={"value": "ek_upstream"}) + return httpx.Response(200, json=self.reply) def client(self) -> AsyncHTTPHandler: return AsyncHTTPHandler(transport=httpx.MockTransport(self)) @@ -55,3 +68,273 @@ async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_k assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls" assert request.headers.get("authorization") == "Bearer ek_upstream" assert "api-key" not in request.headers + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_base", _API_BASES) +@pytest.mark.parametrize(("input_audio_format", "ga_audio_format"), _AUDIO_FORMATS) +async def test_azure_transcription_session_uses_ga_client_secrets( + api_base: str, input_audio_format: str, ga_audio_format: dict[str, str | int] +) -> None: + """Azure format and secret-expiry contracts, checked 2026-10-24. + + https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/azureopenai/realtime#components + """ + azure: Final = _AzureThatRecordsTheRequest( + reply={ + "value": "ek_upstream", + "expires_at": 1792775579, + "session": { + "type": "transcription", + "object": "realtime.transcription_session", + "id": "sess_upstream", + "expires_at": 0, + "audio": { + "input": { + "format": ga_audio_format, + "transcription": {"model": "whisper-deployment", "language": "en"}, + "noise_reduction": {"type": "near_field"}, + } + }, + }, + } + ) + response: Final = await litellm.acreate_realtime_transcription_session( + model="azure/whisper-deployment", + transcription_session={ + "input_audio_format": input_audio_format, + "input_audio_transcription": {"model": "untrusted-model", "language": "en"}, + "input_audio_noise_reduction": {"type": "near_field"}, + }, + api_base=api_base, + api_key="azure-key", + api_version=_DATED_PREVIEW_API_VERSION, + client=azure.client(), + ) + + (request,) = azure.requests + assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/client_secrets" + assert request.headers.get("api-key") == "azure-key" + assert json.loads(request.content) == { + "session": { + "type": "transcription", + "audio": { + "input": { + "format": ga_audio_format, + "transcription": {"model": "whisper-deployment", "language": "en"}, + "noise_reduction": {"type": "near_field"}, + } + }, + } + } + assert response.status_code == 200 + assert response.json() == { + "object": "realtime.transcription_session", + "id": "sess_upstream", + "input_audio_format": input_audio_format, + "input_audio_transcription": {"model": "whisper-deployment", "language": "en"}, + "input_audio_noise_reduction": {"type": "near_field"}, + "client_secret": {"value": "ek_upstream", "expires_at": 1792775579}, + } + + +def test_azure_transcription_client_secret_response_preserves_errors() -> None: + response: Final = httpx.Response(400, json={"error": "bad request"}) + + assert azure_transcription_client_secret_response(response) is response + + +def test_azure_transcription_client_secret_response_preserves_non_object_json() -> None: + response: Final = httpx.Response(200, json=[]) + + assert azure_transcription_client_secret_response(response) is response + + +def test_azure_transcription_client_secret_response_preserves_upstream_headers_and_request() -> None: + request: Final = httpx.Request("GET", "https://my.openai.azure.com/openai/v1/realtime/client_secrets") + payload: Final = { + "value": "ek_upstream", + "expires_at": 1792775579, + "session": { + "type": "transcription", + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": 16000}, + "transcription": {"model": "whisper-deployment", "language": "fr"}, + } + }, + }, + } + compressed_body: Final = gzip.compress(json.dumps(payload).encode()) + upstream_response: Final = httpx.Response( + 200, + headers={ + "content-type": "application/json", + "content-encoding": "gzip", + "content-length": str(len(compressed_body)), + "x-ratelimit-remaining-requests": "17", + }, + content=compressed_body, + request=request, + ) + + response: Final = azure_transcription_client_secret_response(upstream_response) + + assert response.request is request + assert response.headers["x-ratelimit-remaining-requests"] == "17" + assert "content-encoding" not in response.headers + response.raise_for_status() + assert response.json() == { + "input_audio_format": "pcm16", + "input_audio_transcription": {"model": "whisper-deployment", "language": "fr"}, + "client_secret": {"value": "ek_upstream", "expires_at": 1792775579}, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("expiry_style", ("legacy", "ga")) +async def test_azure_transcription_session_preserves_requested_secret_expiry( + expiry_style: Literal["legacy", "ga"], +) -> None: + """Expiry contracts checked 2026-10-24 against the official SDK and Azure reference. + + https://github.com/openai/openai-python/blob/main/src/openai/types/beta/realtime/transcription_session_create_params.py + https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/azureopenai/realtime#openairealtimecreateclientsecretrequest + """ + azure: Final = _AzureThatRecordsTheRequest() + expiry: Final = {"anchor": "created_at", "seconds": 600} + await litellm.acreate_realtime_transcription_session( + model="azure/whisper-deployment", + transcription_session={"client_secret": {"expires_at": expiry}} + if expiry_style == "legacy" + else {"expires_after": expiry}, + api_base="https://my.openai.azure.com", + api_key="azure-key", + client=azure.client(), + ) + + (request,) = azure.requests + assert json.loads(request.content) == { + "expires_after": expiry, + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "whisper-deployment"}}}, + }, + } + + +@pytest.mark.asyncio +async def test_azure_transcription_session_preserves_disabled_audio_processing_with_legacy_modalities() -> None: + """GA transcription has no modalities field and permits null audio settings, checked 2026-10-24. + + https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/azureopenai/realtime#openairealtimetranscriptionsessioncreaterequestga + """ + azure: Final = _AzureThatRecordsTheRequest() + include: Final = ["item.input_audio_transcription.logprobs"] + await litellm.acreate_realtime_transcription_session( + model="azure/whisper-deployment", + transcription_session={ + "modalities": ["text"], + "turn_detection": None, + "input_audio_noise_reduction": None, + "include": include, + }, + api_base="https://my.openai.azure.com", + api_key="azure-key", + client=azure.client(), + ) + + (request,) = azure.requests + assert json.loads(request.content) == { + "session": { + "type": "transcription", + "include": include, + "audio": { + "input": { + "transcription": {"model": "whisper-deployment"}, + "turn_detection": None, + "noise_reduction": None, + } + }, + } + } + + +@pytest.mark.asyncio +async def test_azure_transcription_session_preserves_ga_audio_settings() -> None: + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_transcription_session( + model="azure/whisper-routing-deployment", + transcription_session={ + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": 16000}, + "transcription": {"model": "nested-model", "language": "fr"}, + "noise_reduction": {"type": "near_field"}, + "turn_detection": {"type": "server_vad"}, + }, + "output": {"format": {"type": "audio/pcm", "rate": 24000}}, + } + }, + api_base="https://my.openai.azure.com", + api_key="azure-key", + client=azure.client(), + ) + + (request,) = azure.requests + assert json.loads(request.content) == { + "session": { + "type": "transcription", + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": 16000}, + "transcription": {"model": "whisper-routing-deployment", "language": "fr"}, + "noise_reduction": {"type": "near_field"}, + "turn_detection": {"type": "server_vad"}, + }, + "output": {"format": {"type": "audio/pcm", "rate": 24000}}, + }, + } + } + + +@pytest.mark.asyncio +async def test_azure_transcription_session_legacy_audio_fields_override_ga_settings() -> None: + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_transcription_session( + model="azure/whisper-routing-deployment", + transcription_session={ + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": 16000}, + "transcription": {"model": "nested-model", "language": "fr"}, + "noise_reduction": {"type": "near_field"}, + "turn_detection": {"type": "server_vad"}, + }, + "output": {"format": {"type": "audio/pcm", "rate": 24000}}, + }, + "input_audio_format": "g711_alaw", + "input_audio_transcription": {"language": "de"}, + "input_audio_noise_reduction": None, + "turn_detection": None, + }, + api_base="https://my.openai.azure.com", + api_key="azure-key", + client=azure.client(), + ) + + (request,) = azure.requests + assert json.loads(request.content) == { + "session": { + "type": "transcription", + "audio": { + "input": { + "format": {"type": "audio/pcma"}, + "transcription": {"model": "whisper-routing-deployment", "language": "de"}, + "noise_reduction": None, + "turn_detection": None, + }, + "output": {"format": {"type": "audio/pcm", "rate": 24000}}, + }, + } + }