From 393d9c63cafbc699ea36663cecf688be5dcc4148 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:55:30 -0700 Subject: [PATCH] fix(health): read a realtime gateway's first event and keep Live deployments listing /v1/live healthy Behind a custom api_base the OpenAI realtime health check now waits up to 3 seconds for the first event, so a gateway that passes on OpenAI's invalid_api_key error reports unhealthy while a silent gateway stays healthy. /v1/live counts as a realtime endpoint for the invalid_model rule. --- litellm/llms/openai/realtime/handler.py | 27 ++++++++++++-- litellm/realtime_api/main.py | 24 ++++++++++-- tests/unit/realtime_api/test_main.py | 49 +++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 79032974878..5be488f0f6e 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,6 +4,7 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, cast @@ -47,10 +48,10 @@ def first_event_error(first_event: str | bytes) -> "RealtimeError | None": return RealtimeErrorEvent.model_validate(event).error -async def confirm_session_started( - connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool +def _confirm_first_event( + first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool ) -> Literal[True]: - error: Final = first_event_error(await connection.recv()) + error: Final = first_event_error(first_event) if error is None or ( error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime ): @@ -60,6 +61,26 @@ async def confirm_session_started( raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") +async def confirm_session_started( + connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime) + + +async def confirm_gateway_session_started( + connection: "ClientConnection", + model: str, + sent_credentials: bool, + model_listed_outside_realtime: bool, + first_event_wait_seconds: float, +) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds) + except asyncio.TimeoutError: + return True + return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime) + + class OpenAIRealtime(OpenAIChatCompletion): """ Base handler for OpenAI-compatible realtime WebSocket connections. diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 3fa7d9ae913..e9b534bdd89 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -49,7 +49,11 @@ from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_co from ..llms.custom_httpx.http_handler import realtime_ssl_for_url from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions -from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started +from ..llms.openai.realtime.handler import ( + OpenAIRealtime, + confirm_gateway_session_started, + confirm_session_started, +) from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config @@ -746,7 +750,8 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) -_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions") +_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live") +_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0 def _cost_map_lists_outside_realtime(model: str) -> bool: @@ -802,6 +807,7 @@ async def realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: Mapping[str, object] | None = None, + gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS, ) -> bool: """ Health check for realtime API - tries connection to the realtime API websocket @@ -819,7 +825,9 @@ async def realtime_health_check( Returns: bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports - the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's + the failure in its first server event. Behind any other OpenAI api_base the first event is read for + `gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy. + An `invalid_model` error also counts as healthy unless litellm's cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live and transcription models are served through other endpoints. The listed `mode` is not read, since a deployment's own `mode: realtime` replaces it when the router registers the deployment @@ -920,8 +928,16 @@ async def realtime_health_check( max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=realtime_ssl_for_url(url), ) as connection: - if custom_llm_provider != "openai" or not is_openai_backed_api_base(url): + if custom_llm_provider != "openai": return True + if not is_openai_backed_api_base(url): + return await confirm_gateway_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + first_event_wait_seconds=gateway_first_event_wait_seconds, + ) return await confirm_session_started( connection, model, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 881ecc1bd6e..5478b807691 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -504,13 +504,16 @@ _OPENAI_SESSION_CREATED_EVENT: Final = ( ) -def _register_as_realtime_deployment(model: str) -> None: +def _register_as_realtime_deployment(model: str, *supported_endpoints: str) -> None: litellm.Router( model_list=[ { "model_name": "realtime-deployment", "litellm_params": {"model": f"openai/{model}", "api_key": "sk-real"}, - "model_info": {"mode": "realtime"}, + "model_info": { + "mode": "realtime", + **({"supported_endpoints": list(supported_endpoints)} if supported_endpoints else {}), + }, } ] ) @@ -782,6 +785,18 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real") +@pytest.mark.asyncio +async def test_openai_health_check_is_healthy_for_a_live_deployment_that_lists_the_live_sessions_endpoint( + local_model_cost_map, +): + _register_as_realtime_deployment("gpt-live-1", "/v1/live/sessions") + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT)) + with patch("websockets.connect", connect): + assert await realtime_main.realtime_health_check( + model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real" + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "first_event", "expected_exception", "expected_message"), @@ -815,7 +830,7 @@ async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_w ("deployment_api_base", "global_api_base"), [("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")], ) -async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake( +async def test_openai_health_check_through_a_compatible_gateway_is_healthy_when_it_sends_no_first_event( monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None ): monkeypatch.setattr(litellm, "api_base", global_api_base) @@ -823,13 +838,39 @@ async def test_openai_health_check_through_a_compatible_gateway_trusts_the_hands with patch("websockets.connect", connect): assert await asyncio.wait_for( realtime_main.realtime_health_check( - model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base + model="gpt-realtime", + custom_llm_provider="openai", + api_key="sk-real", + api_base=deployment_api_base, + gateway_first_event_wait_seconds=0.01, ), timeout=5, ) assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("connection", "expected_exception"), + [ + (_ScriptedConnection(_OPENAI_INVALID_API_KEY_EVENT), litellm.AuthenticationError), + (_ConnectionClosedBeforeAnyEvent(), ConnectionClosedError), + ], +) +async def test_openai_health_check_through_a_compatible_gateway_is_unhealthy_when_it_rejects_the_session( + connection: _ServerEvents, expected_exception: type[Exception] +): + connect: Final = _CapturingConnect(connection) + with patch("websockets.connect", connect), pytest.raises(expected_exception): + await realtime_main.realtime_health_check( + model="gpt-realtime", + custom_llm_provider="openai", + api_key="sk-wrong", + api_base="https://realtime-gateway.example/v1", + ) + assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime" + + @pytest.mark.asyncio async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives(): connect: Final = _CapturingConnect(_SilentConnection())