From e56c42862c15d898c0d343adba985fc81bfe6634 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:14:52 -0700 Subject: [PATCH 1/3] fix(health): probe Azure GA realtime path for transcription-only models The realtime health check always built the Azure websocket URL with the default beta protocol, so GA-only transcription models such as azure/gpt-realtime-whisper got probed at /openai/realtime and were rejected with HTTP 400 on every /health run, while real calls through the proxy resolved the GA path via intent=transcription and worked. The probe now resolves the protocol the way the real call path does: an explicit realtime_protocol (argument, deployment litellm_params, or LITELLM_AZURE_REALTIME_PROTOCOL) wins, transcription-only models fall back to GA with intent=transcription, and everything else keeps beta. Transcription-only detection reads both mode and supported_endpoints from get_model_info because a live proxy overwrites the catalog mode with the operator's deployment model_info (mode: realtime) during router registration, while supported_endpoints survives it. get_model_info now propagates supported_endpoints from the cost map; it declared the field but never populated it. --- litellm/realtime_api/main.py | 40 ++++++++- litellm/utils.py | 1 + tests/test_litellm/realtime_api/test_main.py | 92 +++++++++++++++++++- tests/test_litellm/test_utils.py | 9 ++ 4 files changed, 139 insertions(+), 3 deletions(-) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index e5f6c8328f4..d4b9f4e8cce 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -557,6 +557,34 @@ async def _arealtime( raise ValueError(f"Unsupported model: {model}") +def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) -> bool: + try: + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + return False + if model_info.get("mode") == "audio_transcription": + return True + return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) + + +_TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"} + + +def _azure_realtime_health_protocol( + model: str, realtime_protocol: str | None, model_params: Mapping[str, Any] +) -> tuple[str, RealtimeQueryParams | None]: + query_params: Final = _TRANSCRIPTION_QUERY_PARAMS if _is_transcription_only_realtime_model(model, "azure") else None + configured_raw: Final = ( + realtime_protocol or model_params.get("realtime_protocol") or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") + ) + configured: Final = configured_raw if isinstance(configured_raw, str) else None + if configured is not None: + return configured, query_params + if query_params is not None: + return "GA", query_params + return "beta", None + + def _realtime_health_check_auth_headers( custom_llm_provider: str, api_key: str | None, model_params: Mapping[str, Any] ) -> Mapping[str, str | None]: @@ -586,7 +614,9 @@ async def _realtime_health_check( api_version: Optional[str] - api version api_key: str - api key custom_llm_provider: str - custom llm provider - realtime_protocol: Optional[str] - protocol version ("GA"/"v1" for GA path, "beta"/None for beta path) + realtime_protocol: Optional[str] - protocol version ("GA"/"v1" for GA path, "beta" for beta path); + None resolves it for Azure from model_params/env, with transcription-only models probing GA + plus intent=transcription the way real calls do Returns: bool - True if connection is successful, False otherwise @@ -602,11 +632,17 @@ async def _realtime_health_check( model_params=model_params or _EMPTY_MODEL_PARAMS, ) if custom_llm_provider == "azure": + resolved_protocol, azure_query_params = _azure_realtime_health_protocol( + model=model, + realtime_protocol=realtime_protocol, + model_params=model_params or _EMPTY_MODEL_PARAMS, + ) url = azure_realtime._construct_url( api_base=api_base or "", model=model, api_version=api_version or "2024-10-01-preview", - realtime_protocol=realtime_protocol, + realtime_protocol=resolved_protocol, + query_params=azure_query_params, ) elif custom_llm_provider == "openai": url = openai_realtime._construct_url( diff --git a/litellm/utils.py b/litellm/utils.py index 1b672018507..c2770a1a26d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5845,6 +5845,7 @@ def _get_model_info_helper( tiered_pricing=_model_info.get("tiered_pricing", None), litellm_provider=_model_info.get("litellm_provider", custom_llm_provider), mode=_model_info.get("mode"), + supported_endpoints=_model_info.get("supported_endpoints", None), supports_system_messages=_model_info.get("supports_system_messages", None), supports_response_schema=_model_info.get("supports_response_schema", None), supports_vision=_model_info.get("supports_vision", None), diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index a0c0d849e3e..4f50bbbd8c5 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -1,10 +1,11 @@ import asyncio import time -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest +import litellm from litellm.realtime_api import main as realtime_main from litellm.realtime_api.main import _with_resolved_session_model @@ -190,3 +191,92 @@ def test_client_secret_forwards_nested_transcription_model_untouched(monkeypatch session = captured["request_data"]["session"] assert session["model"] == "gpt-4o-realtime-preview" assert session["input_audio_transcription"]["model"] == "whisper-1" + + +class _CapturingConnect: + def __init__(self): + self.url = None + self.additional_headers = None + + def __call__(self, url, **kwargs): + self.url = url + self.additional_headers = kwargs.get("additional_headers") + return self + + async def __aenter__(self): + return MagicMock() + + async def __aexit__(self, exc_type, exc, tb): + return None + + +@pytest.mark.asyncio +async def test_azure_health_check_probes_ga_transcription_url_for_transcription_model(local_model_cost_map): + """Regression for LIT-6240: transcription-only models (mode audio_transcription + in the cost map) are GA-only and 400 on the beta path, so the health probe + must hit /openai/v1/realtime?intent=transcription like real calls do.""" + connect = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime-whisper", + custom_llm_provider="azure", + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version="2025-04-01-preview", + ) + assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?intent=transcription" + + +@pytest.mark.asyncio +async def test_azure_health_check_stays_on_ga_when_deployment_registration_overwrites_mode( + local_model_cost_map, monkeypatch +): + """In a live proxy, Router._register_deployment_in_model_cost writes the + operator's deployment model_info (mode: realtime) over the catalog entry for + azure/gpt-realtime-whisper, so mode alone misreads the model as speech-capable + and the probe regresses to the beta path. supported_endpoints survives that + registration and must keep the probe on the GA transcription path.""" + polluted = {**litellm.model_cost["azure/gpt-realtime-whisper"], "mode": "realtime"} + monkeypatch.setitem(litellm.model_cost, "azure/gpt-realtime-whisper", polluted) + connect = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime-whisper", + custom_llm_provider="azure", + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version="2025-04-01-preview", + ) + assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?intent=transcription" + + +@pytest.mark.asyncio +async def test_azure_health_check_keeps_beta_path_for_speech_model(): + connect = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-4o-realtime-preview", + custom_llm_provider="azure", + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version="2024-10-01-preview", + ) + assert connect.url == ( + "wss://my-endpoint.openai.azure.com/openai/realtime" + "?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview" + ) + + +@pytest.mark.asyncio +async def test_azure_health_check_honors_deployment_realtime_protocol(): + connect = _CapturingConnect() + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-4o-realtime-preview", + custom_llm_provider="azure", + api_key="fake-key", + api_base="https://my-endpoint.openai.azure.com", + api_version="2024-10-01-preview", + model_params={"realtime_protocol": "GA"}, + ) + assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-4o-realtime-preview" diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index e1ea6edb2d8..ecf9ccd8037 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -120,6 +120,15 @@ def test_get_model_info_surfaces_supports_adaptive_thinking(local_model_cost_map assert generalized["supports_adaptive_thinking"] is True +def test_get_model_info_surfaces_supported_endpoints(local_model_cost_map): + """supported_endpoints ships in the cost map and is declared on ModelInfoBase, + but the constructor never copied it, so get_model_info always returned None. + The realtime health check reads it to spot GA-only transcription models + (LIT-6240).""" + info = litellm.get_model_info(model="gpt-realtime-whisper", custom_llm_provider="azure") + assert info["supported_endpoints"] == ["/v1/realtime", "/v1/realtime/transcription_sessions"] + + def test_potential_model_names_keeps_provider_prefixed_candidate(): """A provider whose own model ids repeat the litellm provider name (Perplexity's Agent API serves `perplexity/glm-5.2`, mapped as `perplexity/perplexity/glm-5.2`) From 3c9690c4f57cf21f25341d26557326ffc202070b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:27:05 -0700 Subject: [PATCH 2/3] test(realtime): fully type the capturing websocket connect double --- tests/test_litellm/realtime_api/test_main.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index 4f50bbbd8c5..4b0bae1458e 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -1,5 +1,6 @@ import asyncio import time +from types import TracebackType from unittest.mock import MagicMock, patch @@ -194,19 +195,22 @@ def test_client_secret_forwards_nested_transcription_model_untouched(monkeypatch class _CapturingConnect: - def __init__(self): - self.url = None - self.additional_headers = None + def __init__(self) -> None: + self.url: str | None = None - def __call__(self, url, **kwargs): + def __call__(self, url: str, **kwargs: object) -> "_CapturingConnect": self.url = url - self.additional_headers = kwargs.get("additional_headers") return self - async def __aenter__(self): + async def __aenter__(self) -> MagicMock: return MagicMock() - async def __aexit__(self, exc_type, exc, tb): + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: return None From 7c717c7c6a2f172fcdbd0a5f64a22417d9d15983 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:33:46 -0700 Subject: [PATCH 3/3] test(realtime): pin mode-only transcription detection and correct a stale docstring --- .../test_github_copilot_responses_transformation.py | 7 +++---- tests/test_litellm/realtime_api/test_main.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py b/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py index c761d084da8..0174465b0cc 100644 --- a/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py +++ b/tests/test_litellm/llms/github_copilot/responses/test_github_copilot_responses_transformation.py @@ -436,10 +436,9 @@ class TestGithubCopilotResponsesAPIRouting: catalog entries that lack ``mode``). Exercises the real ``_cached_get_model_info_helper`` plumbing via - ``register_model`` (no mock). ``supported_endpoints`` is not carried on - the normalized ``ModelInfoBase`` the helper returns, so the gate must - read it from the raw ``litellm.model_cost`` entry; a mock-based test - would mask that. + ``register_model`` (no mock). The gate reads ``supported_endpoints`` + from the raw ``litellm.model_cost`` entry; a mock-based test would + mask that. """ litellm.register_model( { diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index 4b0bae1458e..a3dd5688ad1 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -254,6 +254,16 @@ async def test_azure_health_check_stays_on_ga_when_deployment_registration_overw assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?intent=transcription" +def test_transcription_only_detection_falls_back_to_mode(local_model_cost_map): + """azure/whisper-1 declares mode audio_transcription but no supported_endpoints, + so only the mode signal can classify it as transcription-only.""" + assert realtime_main._is_transcription_only_realtime_model("whisper-1", "azure") is True + + +def test_transcription_only_detection_rejects_speech_model(local_model_cost_map): + assert realtime_main._is_transcription_only_realtime_model("gpt-realtime-mini", "azure") is False + + @pytest.mark.asyncio async def test_azure_health_check_keeps_beta_path_for_speech_model(): connect = _CapturingConnect()