scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 233

Merge 133457b9b9 into 6063187fea

Browse source
This commit is contained in:
shrey 2026-10-28 15:55:26 -07:00 • committed by GitHub
commit 531dcfe86a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 429 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

126
litellm/llms/azure/realtime/transcription_sessions.py Normal file
View file

@ -0,0 +1,126 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import JsonValue, TypeAdapter
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_AUDIO_FORMATS: Final[Mapping[str, JsonValue]] = MappingProxyType(
{
"pcm16": {"type": "audio/pcm", "rate": 24000},
"g711_ulaw": {"type": "audio/pcmu"},
"g711_alaw": {"type": "audio/pcma"},
}
)
_LEGACY_AUDIO_FORMATS: Final[Mapping[str, str]] = MappingProxyType(
{"audio/pcm": "pcm16", "audio/pcmu": "g711_ulaw", "audio/pcma": "g711_alaw"}
)
def azure_transcription_client_secret_request(
transcription_session: Mapping[str, JsonValue], model: str
) -> Mapping[str, JsonValue]:
raw_secret: Final = transcription_session.get("client_secret")
expires_after: Final = transcription_session.get(
"expires_after", raw_secret.get("expires_at") if isinstance(raw_secret, dict) else raw_secret
)
raw_audio: Final = transcription_session.get("audio")
audio: Final = raw_audio if isinstance(raw_audio, dict) else {}
raw_audio_input: Final = audio.get("input")
existing_audio_input: Final = raw_audio_input if isinstance(raw_audio_input, dict) else {}
raw_audio_transcription: Final = existing_audio_input.get("transcription")
audio_transcription: Final = raw_audio_transcription if isinstance(raw_audio_transcription, dict) else {}
raw_transcription: Final = transcription_session.get("input_audio_transcription")
transcription: Final = raw_transcription if isinstance(raw_transcription, dict) else {}
raw_format: Final = transcription_session.get("input_audio_format")
audio_input: Final[JsonValue] = {
**existing_audio_input,
**(
{
"format": _AUDIO_FORMATS[raw_format]
if isinstance(raw_format, str) and raw_format in _AUDIO_FORMATS
else raw_format
}
if "input_audio_format" in transcription_session
else {}
),
"transcription": {**audio_transcription, **transcription, "model": model},
**(
{"noise_reduction": transcription_session["input_audio_noise_reduction"]}
if "input_audio_noise_reduction" in transcription_session
else {}
),
**(
{"turn_detection": transcription_session["turn_detection"]}
if "turn_detection" in transcription_session
else {}
),
}
session: Final[JsonValue] = {
**{
key: value
for key, value in transcription_session.items()
if key
not in (
"model",
"audio",
"client_secret",
"expires_after",
"modalities",
"input_audio_format",
"input_audio_transcription",
"input_audio_noise_reduction",
"turn_detection",
)
},
"type": "transcription",
"audio": {
**{key: value for key, value in audio.items() if key != "input"},
"input": audio_input,
},
}
return MappingProxyType(
{"session": session, **({"expires_after": expires_after} if expires_after is not None else {})}
)
def azure_transcription_client_secret_response(response: httpx.Response) -> httpx.Response:
if response.is_error:
return response
payload: Final = _JSON.validate_python(response.json())
if not isinstance(payload, dict):
return response
raw_session: Final = payload.get("session")
session: Final = raw_session if isinstance(raw_session, dict) else {}
raw_audio: Final = session.get("audio")
audio: Final = raw_audio if isinstance(raw_audio, dict) else {}
raw_input: Final = audio.get("input")
audio_input: Final = raw_input if isinstance(raw_input, dict) else {}
raw_format: Final = audio_input.get("format")
format_type: Final = raw_format.get("type") if isinstance(raw_format, dict) else None
input_audio_format: Final = (
_LEGACY_AUDIO_FORMATS.get(format_type, format_type) if isinstance(format_type, str) else raw_format
)
client_secret: Final = {
"value": payload.get("value"),
**({"expires_at": payload["expires_at"]} if "expires_at" in payload else {}),
}
response_body: Final = {
**{key: value for key, value in session.items() if key not in ("type", "audio", "expires_at")},
**({"input_audio_format": input_audio_format} if input_audio_format is not None else {}),
**({"input_audio_transcription": audio_input["transcription"]} if "transcription" in audio_input else {}),
**({"input_audio_noise_reduction": audio_input["noise_reduction"]} if "noise_reduction" in audio_input else {}),
**({"turn_detection": audio_input["turn_detection"]} if "turn_detection" in audio_input else {}),
"client_secret": client_secret,
}
return httpx.Response(
response.status_code,
headers={
key: value
for key, value in response.headers.items()
if key.lower() not in ("content-length", "content-encoding", "transfer-encoding")
},
json=response_body,
request=response.request,
)

19
litellm/realtime_api/main.py
View file

@ -40,6 +40,10 @@ from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from ..litellm_core_utils.realtime_streaming import client_websocket_scope
from ..llms.azure.common_utils import get_azure_ad_token
from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client
from ..llms.azure.realtime.transcription_sessions import (
azure_transcription_client_secret_request,
azure_transcription_client_secret_response,
)
from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed
from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
from ..llms.openai.live.handler import OpenAILiveSessions
@ -267,19 +271,27 @@ async def acreate_realtime_transcription_session(
litellm_params={"api_base": resolved_api_base},
custom_llm_provider=custom_llm_provider,
)
request_data: Final = req.model_dump(exclude_none=True, exclude={"model"})
is_azure: Final = custom_llm_provider == LlmProviders.AZURE.value
request_data: Final = req.model_dump(exclude_none=not is_azure, exclude_unset=is_azure, exclude={"model"})
# Ensure the upstream body's input_audio_transcription.model matches the
# authorized routing model. This prevents a caller from supplying an allowed
# top-level model for auth while sneaking a different model into the nested
# transcription config that gets forwarded to the provider.
if isinstance(request_data.get("input_audio_transcription"), dict):
request_data["input_audio_transcription"]["model"] = model_name
return await base_llm_http_handler.async_realtime_transcription_session_handler(
create_session: Final = (
base_llm_http_handler.async_realtime_client_secret_handler
if is_azure
else base_llm_http_handler.async_realtime_transcription_session_handler
)
response: Final = await create_session(
api_base=resolved_api_base,
api_key=await _realtime_http_bearer_token(
custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params
),
request_data=request_data,
request_data=dict(azure_transcription_client_secret_request(request_data, model_name))
if is_azure
else request_data,
logging_obj=litellm_logging_obj,
timeout=timeout or request_timeout,
provider_config=provider_config,
@ -288,6 +300,7 @@ async def acreate_realtime_transcription_session(
client=kwargs.get("client"),
api_version=litellm_params.api_version,
)
return azure_transcription_client_secret_response(response) if is_azure else response
@wrapper_client

291
tests/unit/llms/azure/realtime/test_http_transformation.py
View file

@ -1,22 +1,35 @@
from typing import Final
import gzip
import json
from typing import Final, Literal
import httpx
import pytest
import litellm
from litellm.llms.azure.realtime.transcription_sessions import azure_transcription_client_secret_response
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
_DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview"
_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai")
_API_BASES: Final = (
"https://my.openai.azure.com",
"https://my.openai.azure.com/",
"https://my.openai.azure.com/openai",
)
_AUDIO_FORMATS: Final = (
("pcm16", {"type": "audio/pcm", "rate": 24000}),
("g711_ulaw", {"type": "audio/pcmu"}),
("g711_alaw", {"type": "audio/pcma"}),
)
class _AzureThatRecordsTheRequest:
def __init__(self) -> None:
def __init__(self, reply: dict[str, object] | None = None) -> None:
self.requests: Final[list[httpx.Request]] = []
self.reply: Final = reply or {"value": "ek_upstream"}
def __call__(self, request: httpx.Request) -> httpx.Response:
self.requests.append(request)
return httpx.Response(200, json={"value": "ek_upstream"})
return httpx.Response(200, json=self.reply)
def client(self) -> AsyncHTTPHandler:
return AsyncHTTPHandler(transport=httpx.MockTransport(self))
@ -55,3 +68,273 @@ async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_k
assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls"
assert request.headers.get("authorization") == "Bearer ek_upstream"
assert "api-key" not in request.headers
@pytest.mark.asyncio
@pytest.mark.parametrize("api_base", _API_BASES)
@pytest.mark.parametrize(("input_audio_format", "ga_audio_format"), _AUDIO_FORMATS)
async def test_azure_transcription_session_uses_ga_client_secrets(
api_base: str, input_audio_format: str, ga_audio_format: dict[str, str | int]
) -> None:
"""Azure format and secret-expiry contracts, checked 2026-10-24.
https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/azureopenai/realtime#components
"""
azure: Final = _AzureThatRecordsTheRequest(
reply={
"value": "ek_upstream",
"expires_at": 1792775579,
"session": {
"type": "transcription",
"object": "realtime.transcription_session",
"id": "sess_upstream",
"expires_at": 0,
"audio": {
"input": {
"format": ga_audio_format,
"transcription": {"model": "whisper-deployment", "language": "en"},
"noise_reduction": {"type": "near_field"},
}
},
},
}
)
response: Final = await litellm.acreate_realtime_transcription_session(
model="azure/whisper-deployment",
transcription_session={
"input_audio_format": input_audio_format,
"input_audio_transcription": {"model": "untrusted-model", "language": "en"},
"input_audio_noise_reduction": {"type": "near_field"},
},
api_base=api_base,
api_key="azure-key",
api_version=_DATED_PREVIEW_API_VERSION,
client=azure.client(),
)
(request,) = azure.requests
assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/client_secrets"
assert request.headers.get("api-key") == "azure-key"
assert json.loads(request.content) == {
"session": {
"type": "transcription",
"audio": {
"input": {
"format": ga_audio_format,
"transcription": {"model": "whisper-deployment", "language": "en"},
"noise_reduction": {"type": "near_field"},
}
},
}
}
assert response.status_code == 200
assert response.json() == {
"object": "realtime.transcription_session",
"id": "sess_upstream",
"input_audio_format": input_audio_format,
"input_audio_transcription": {"model": "whisper-deployment", "language": "en"},
"input_audio_noise_reduction": {"type": "near_field"},
"client_secret": {"value": "ek_upstream", "expires_at": 1792775579},
}
def test_azure_transcription_client_secret_response_preserves_errors() -> None:
response: Final = httpx.Response(400, json={"error": "bad request"})
assert azure_transcription_client_secret_response(response) is response
def test_azure_transcription_client_secret_response_preserves_non_object_json() -> None:
response: Final = httpx.Response(200, json=[])
assert azure_transcription_client_secret_response(response) is response
def test_azure_transcription_client_secret_response_preserves_upstream_headers_and_request() -> None:
request: Final = httpx.Request("GET", "https://my.openai.azure.com/openai/v1/realtime/client_secrets")
payload: Final = {
"value": "ek_upstream",
"expires_at": 1792775579,
"session": {
"type": "transcription",
"audio": {
"input": {
"format": {"type": "audio/pcm", "rate": 16000},
"transcription": {"model": "whisper-deployment", "language": "fr"},
}
},
},
}
compressed_body: Final = gzip.compress(json.dumps(payload).encode())
upstream_response: Final = httpx.Response(
200,
headers={
"content-type": "application/json",
"content-encoding": "gzip",
"content-length": str(len(compressed_body)),
"x-ratelimit-remaining-requests": "17",
},
content=compressed_body,
request=request,
)
response: Final = azure_transcription_client_secret_response(upstream_response)
assert response.request is request
assert response.headers["x-ratelimit-remaining-requests"] == "17"
assert "content-encoding" not in response.headers
response.raise_for_status()
assert response.json() == {
"input_audio_format": "pcm16",
"input_audio_transcription": {"model": "whisper-deployment", "language": "fr"},
"client_secret": {"value": "ek_upstream", "expires_at": 1792775579},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("expiry_style", ("legacy", "ga"))
async def test_azure_transcription_session_preserves_requested_secret_expiry(
expiry_style: Literal["legacy", "ga"],
) -> None:
"""Expiry contracts checked 2026-10-24 against the official SDK and Azure reference.
https://github.com/openai/openai-python/blob/main/src/openai/types/beta/realtime/transcription_session_create_params.py
https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/azureopenai/realtime#openairealtimecreateclientsecretrequest
"""
azure: Final = _AzureThatRecordsTheRequest()
expiry: Final = {"anchor": "created_at", "seconds": 600}
await litellm.acreate_realtime_transcription_session(
model="azure/whisper-deployment",
transcription_session={"client_secret": {"expires_at": expiry}}
if expiry_style == "legacy"
else {"expires_after": expiry},
api_base="https://my.openai.azure.com",
api_key="azure-key",
client=azure.client(),
)
(request,) = azure.requests
assert json.loads(request.content) == {
"expires_after": expiry,
"session": {
"type": "transcription",
"audio": {"input": {"transcription": {"model": "whisper-deployment"}}},
},
}
@pytest.mark.asyncio
async def test_azure_transcription_session_preserves_disabled_audio_processing_with_legacy_modalities() -> None:
"""GA transcription has no modalities field and permits null audio settings, checked 2026-10-24.
https://learn.microsoft.com/en-us/rest/api/microsoft-foundry/azureopenai/realtime#openairealtimetranscriptionsessioncreaterequestga
"""
azure: Final = _AzureThatRecordsTheRequest()
include: Final = ["item.input_audio_transcription.logprobs"]
await litellm.acreate_realtime_transcription_session(
model="azure/whisper-deployment",
transcription_session={
"modalities": ["text"],
"turn_detection": None,
"input_audio_noise_reduction": None,
"include": include,
},
api_base="https://my.openai.azure.com",
api_key="azure-key",
client=azure.client(),
)
(request,) = azure.requests
assert json.loads(request.content) == {
"session": {
"type": "transcription",
"include": include,
"audio": {
"input": {
"transcription": {"model": "whisper-deployment"},
"turn_detection": None,
"noise_reduction": None,
}
},
}
}
@pytest.mark.asyncio
async def test_azure_transcription_session_preserves_ga_audio_settings() -> None:
azure: Final = _AzureThatRecordsTheRequest()
await litellm.acreate_realtime_transcription_session(
model="azure/whisper-routing-deployment",
transcription_session={
"audio": {
"input": {
"format": {"type": "audio/pcm", "rate": 16000},
"transcription": {"model": "nested-model", "language": "fr"},
"noise_reduction": {"type": "near_field"},
"turn_detection": {"type": "server_vad"},
},
"output": {"format": {"type": "audio/pcm", "rate": 24000}},
}
},
api_base="https://my.openai.azure.com",
api_key="azure-key",
client=azure.client(),
)
(request,) = azure.requests
assert json.loads(request.content) == {
"session": {
"type": "transcription",
"audio": {
"input": {
"format": {"type": "audio/pcm", "rate": 16000},
"transcription": {"model": "whisper-routing-deployment", "language": "fr"},
"noise_reduction": {"type": "near_field"},
"turn_detection": {"type": "server_vad"},
},
"output": {"format": {"type": "audio/pcm", "rate": 24000}},
},
}
}
@pytest.mark.asyncio
async def test_azure_transcription_session_legacy_audio_fields_override_ga_settings() -> None:
azure: Final = _AzureThatRecordsTheRequest()
await litellm.acreate_realtime_transcription_session(
model="azure/whisper-routing-deployment",
transcription_session={
"audio": {
"input": {
"format": {"type": "audio/pcm", "rate": 16000},
"transcription": {"model": "nested-model", "language": "fr"},
"noise_reduction": {"type": "near_field"},
"turn_detection": {"type": "server_vad"},
},
"output": {"format": {"type": "audio/pcm", "rate": 24000}},
},
"input_audio_format": "g711_alaw",
"input_audio_transcription": {"language": "de"},
"input_audio_noise_reduction": None,
"turn_detection": None,
},
api_base="https://my.openai.azure.com",
api_key="azure-key",
client=azure.client(),
)
(request,) = azure.requests
assert json.loads(request.content) == {
"session": {
"type": "transcription",
"audio": {
"input": {
"format": {"type": "audio/pcma"},
"transcription": {"model": "whisper-routing-deployment", "language": "de"},
"noise_reduction": None,
"turn_detection": None,
},
"output": {"format": {"type": "audio/pcm", "rate": 24000}},
},
}
}