From 8a0da671d64a763f4541235452d6659704713593 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 15 Sep 2026 05:30:02 -0700 Subject: [PATCH 1/7] fix(health): report openai realtime deployments with a missing or invalid api key as unhealthy --- litellm/constants.py | 1 + litellm/realtime_api/main.py | 68 ++++++++-- tests/test_litellm/realtime_api/test_main.py | 126 ++++++++++++++++++- 3 files changed, 181 insertions(+), 14 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 09442d6151e..90d1311dd74 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -310,6 +310,7 @@ REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = int(_max_size_env) if _max_si REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float( os.getenv("REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", "20.0") ) +REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS: Final = 10.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 44c47af57f4..937cf197d41 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -6,10 +6,14 @@ from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast +from openai.types.realtime import RealtimeError, RealtimeErrorEvent +from pydantic import TypeAdapter + import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, + REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) @@ -44,6 +48,7 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket + from websockets.asyncio.client import ClientConnection from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig @@ -54,6 +59,7 @@ xai_realtime: Final = XAIRealtime() vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() _EMPTY_MODEL_PARAMS: Final[Mapping[str, Any]] = MappingProxyType({}) +_EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: @@ -591,13 +597,48 @@ def _azure_realtime_health_protocol( def _realtime_health_check_auth_headers( custom_llm_provider: str, api_key: str | None, model_params: Mapping[str, Any] -) -> Mapping[str, str | None]: - if custom_llm_provider != "azure": - return MappingProxyType({"api-key": api_key}) - return azure_realtime.get_auth_headers( - api_key=api_key, - azure_ad_token=(None if api_key else get_azure_ad_token(GenericLiteLLMParams(**model_params))), - ) +) -> Mapping[str, str]: + if custom_llm_provider == "azure": + return azure_realtime.get_auth_headers( + api_key=api_key, + azure_ad_token=(None if api_key else get_azure_ad_token(GenericLiteLLMParams(**model_params))), + ) + if api_key is None: + return _EMPTY_AUTH_HEADERS + return MappingProxyType({"Authorization": f"Bearer {api_key}"}) + + +_REALTIME_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def _realtime_first_event_error(first_event: str | bytes) -> RealtimeError | None: + event: Final = _REALTIME_SERVER_EVENT_FIELDS.validate_json(first_event) + if event.get("type") != "error": + return None + return RealtimeErrorEvent.model_validate(event).error + + +def _realtime_first_event_exception(error: RealtimeError, model: str) -> Exception: + match error: + case RealtimeError(code="invalid_api_key"): + return litellm.AuthenticationError(message=error.message, llm_provider="openai", model=model) + case RealtimeError(type="server_error"): + return litellm.InternalServerError(message=error.message, llm_provider="openai", model=model) + case _: + return litellm.BadRequestError(message=error.message, model=model, llm_provider="openai") + + +async def _confirm_realtime_session_started( + connection: "ClientConnection", model: str, timeout_seconds: float +) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) + except asyncio.TimeoutError: + return True + error: Final = _realtime_first_event_error(first_event) + if error is None: + return True + raise _realtime_first_event_exception(error, model) async def _realtime_health_check( @@ -608,6 +649,7 @@ async def _realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: dict | None = None, + first_event_timeout_seconds: float = REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, ): """ Health check for realtime API - tries connection to the realtime API websocket @@ -623,9 +665,11 @@ async def _realtime_health_check( without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription Returns: - bool - True if connection is successful, False otherwise + bool - True once the connection is open, and for OpenAI once the first server event is not an error, + since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports + the failure in its first server event Raises: - Exception - if the connection is not successful + Exception - if the connection is not successful, or if OpenAI's first server event is an error """ import websockets @@ -693,5 +737,7 @@ async def _realtime_health_check( additional_headers=auth_headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=ssl_context, - ): - return True + ) as connection: + if custom_llm_provider != "openai": + return True + return await _confirm_realtime_session_started(connection, model, first_event_timeout_seconds) diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index d3d41c5b54b..bbf7ebfbe3b 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -5,6 +5,8 @@ from typing import Final from unittest.mock import MagicMock, patch import pytest +from websockets.exceptions import ConnectionClosedError +from websockets.frames import Close import litellm from litellm.realtime_api import main as realtime_main @@ -222,15 +224,18 @@ def test_client_secret_forwards_nested_transcription_model_untouched(monkeypatch class _CapturingConnect: - def __init__(self) -> None: + def __init__(self, connection: object | None = None) -> None: self.url: str | None = None + self.kwargs: dict[str, object] = {} + self._connection: Final = connection if connection is not None else MagicMock() def __call__(self, url: str, **kwargs: object) -> "_CapturingConnect": self.url = url + self.kwargs = kwargs return self - async def __aenter__(self) -> MagicMock: - return MagicMock() + async def __aenter__(self) -> object: + return self._connection async def __aexit__( self, @@ -343,6 +348,121 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch): assert connect.url == _AZURE_BETA_HEALTH_URL +class _ScriptedConnection: + def __init__(self, *frames: str) -> None: + self._frames: Final = iter(frames) + + async def recv(self) -> str: + return next(self._frames) + + +class _SilentConnection: + async def recv(self) -> str: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +class _ConnectionClosedBeforeAnyEvent: + async def recv(self) -> str: + raise ConnectionClosedError(Close(3000, "invalid_api_key"), None) + + +_OPENAI_INVALID_API_KEY_EVENT: Final = ( + '{"type": "error", "event_id": "event_1", "error": {"type": "invalid_request_error", "code": "invalid_api_key", ' + '"message": "Incorrect API key provided: sk-proj-****0000. You can find your API key at ' + 'https://platform.openai.com/account/api-keys.", "param": null, "event_id": null}}' +) +_OPENAI_MISSING_AUTH_EVENT: Final = ( + '{"type": "error", "event_id": "event_2", "error": {"type": "invalid_request_error", "code": null, ' + '"message": "Missing bearer or basic authentication in header", "param": null, "event_id": null}}' +) +_OPENAI_SERVER_ERROR_EVENT: Final = ( + '{"type": "error", "event_id": "event_3", ' + '"error": {"type": "server_error", "code": null, "message": "The server had an error", "param": null}}' +) +_OPENAI_SESSION_CREATED_EVENT: Final = ( + '{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}' +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("first_event", "expected_exception", "expected_status", "expected_message"), + [ + (_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"), + (_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"), + (_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"), + ], +) +async def test_openai_health_check_reports_the_first_error_event_as_unhealthy( + first_event: str, expected_exception: type[Exception], expected_status: int, expected_message: str +): + connect: Final = _CapturingConnect(_ScriptedConnection(first_event)) + with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised: + await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-wrong" + ) + assert raised.value.status_code == expected_status + assert expected_message in str(raised.value) + assert connect.url == "wss://api.openai.com/v1/realtime?model=gpt-realtime" + + +@pytest.mark.asyncio +async def test_openai_health_check_sends_the_api_key_as_a_bearer_token(): + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_SESSION_CREATED_EVENT)) + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real" + ) + assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer sk-real"} + + +@pytest.mark.asyncio +async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_is_unhealthy(): + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_MISSING_AUTH_EVENT)) + with patch("websockets.connect", connect), pytest.raises(litellm.BadRequestError) as raised: + await realtime_main._realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key=None) + assert connect.kwargs["additional_headers"] == {} + assert "Missing bearer or basic authentication" in str(raised.value) + + +@pytest.mark.asyncio +async def test_openai_health_check_is_healthy_once_session_created_arrives(): + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_SESSION_CREATED_EVENT)) + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real" + ) + + +@pytest.mark.asyncio +async def test_openai_health_check_stays_healthy_when_no_first_event_arrives_in_time(): + connect: Final = _CapturingConnect(_SilentConnection()) + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01 + ) + + +@pytest.mark.asyncio +async def test_openai_health_check_is_unhealthy_when_the_socket_closes_before_any_event(): + connect: Final = _CapturingConnect(_ConnectionClosedBeforeAnyEvent()) + with patch("websockets.connect", connect), pytest.raises(ConnectionClosedError): + await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-wrong" + ) + + +@pytest.mark.asyncio +async def test_xai_health_check_trusts_the_handshake_without_reading_a_first_event(): + connect: Final = _CapturingConnect(_ScriptedConnection()) + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="grok-voice-latest", custom_llm_provider="xai", api_key="sk-wrong" + ) + assert connect.url == "wss://api.x.ai/v1/realtime?model=grok-voice-latest" + + class _ConnectThatStopsAfterCapturingTheUrl: url: str | None = None From a7c2be768542e6ecdd338f8c41278a11bc578935 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:39:03 -0700 Subject: [PATCH 2/7] fix(health): time out a silent openai realtime handshake and keep the first-event parsing in the openai handler --- litellm/llms/openai/realtime/handler.py | 45 +++++++++++++++++++- litellm/realtime_api/main.py | 44 ++----------------- tests/test_litellm/realtime_api/test_main.py | 8 ++-- 3 files changed, 53 insertions(+), 44 deletions(-) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index e3ecbac1a53..cdac99721e5 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl -from typing import Any, Final, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, Literal, cast + +from openai.types.realtime import RealtimeError, RealtimeErrorEvent +from pydantic import TypeAdapter from litellm._logging import _redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.exceptions import AuthenticationError, BadRequestError, InternalServerError, Timeout from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -20,6 +26,43 @@ from ....litellm_core_utils.realtime_streaming import ( from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..openai import OpenAIChatCompletion +if TYPE_CHECKING: + from websockets.asyncio.client import ClientConnection + +_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def first_event_error(first_event: str | bytes) -> RealtimeError | None: + event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) + if event.get("type") != "error": + return None + return RealtimeErrorEvent.model_validate(event).error + + +def first_event_exception(error: RealtimeError, model: str) -> Exception: + match error: + case RealtimeError(code="invalid_api_key"): + return AuthenticationError(message=error.message, llm_provider="openai", model=model) + case RealtimeError(type="server_error"): + return InternalServerError(message=error.message, llm_provider="openai", model=model) + case _: + return BadRequestError(message=error.message, model=model, llm_provider="openai") + + +async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) + except asyncio.TimeoutError: + raise Timeout( + message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake", + model=model, + llm_provider="openai", + ) from None + error: Final = first_event_error(first_event) + if error is None: + return True + raise first_event_exception(error, model) + class OpenAIRealtime(OpenAIChatCompletion): """ diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index c24c34e19c9..1b87c7a1eea 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -6,9 +6,6 @@ from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast -from openai.types.realtime import RealtimeError, RealtimeErrorEvent -from pydantic import TypeAdapter - import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, @@ -41,7 +38,7 @@ from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime from ..llms.custom_httpx.http_handler import get_shared_realtime_ssl_context -from ..llms.openai.realtime.handler import OpenAIRealtime +from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig from ..llms.vertex_ai.vertex_llm_base import VertexBase from ..llms.xai.realtime.handler import XAIRealtime @@ -49,7 +46,6 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket - from websockets.asyncio.client import ClientConnection from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig @@ -619,39 +615,6 @@ def _realtime_health_check_auth_headers( return MappingProxyType({"Authorization": f"Bearer {api_key}"}) -_REALTIME_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) - - -def _realtime_first_event_error(first_event: str | bytes) -> RealtimeError | None: - event: Final = _REALTIME_SERVER_EVENT_FIELDS.validate_json(first_event) - if event.get("type") != "error": - return None - return RealtimeErrorEvent.model_validate(event).error - - -def _realtime_first_event_exception(error: RealtimeError, model: str) -> Exception: - match error: - case RealtimeError(code="invalid_api_key"): - return litellm.AuthenticationError(message=error.message, llm_provider="openai", model=model) - case RealtimeError(type="server_error"): - return litellm.InternalServerError(message=error.message, llm_provider="openai", model=model) - case _: - return litellm.BadRequestError(message=error.message, model=model, llm_provider="openai") - - -async def _confirm_realtime_session_started( - connection: "ClientConnection", model: str, timeout_seconds: float -) -> Literal[True]: - try: - first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) - except asyncio.TimeoutError: - return True - error: Final = _realtime_first_event_error(first_event) - if error is None: - return True - raise _realtime_first_event_exception(error, model) - - async def _realtime_health_check( model: str, custom_llm_provider: str, @@ -680,7 +643,8 @@ async def _realtime_health_check( since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports the failure in its first server event Raises: - Exception - if the connection is not successful, or if OpenAI's first server event is an error + Exception - if the connection is not successful, if OpenAI's first server event is an error, or if + OpenAI sends no server event within first_event_timeout_seconds """ import websockets @@ -763,4 +727,4 @@ async def _realtime_health_check( ) as connection: if custom_llm_provider != "openai": return True - return await _confirm_realtime_session_started(connection, model, first_event_timeout_seconds) + return await confirm_session_started(connection, model, first_event_timeout_seconds) diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index 8213e2bdf3e..911f9c74891 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -493,12 +493,14 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives(): @pytest.mark.asyncio -async def test_openai_health_check_stays_healthy_when_no_first_event_arrives_in_time(): +async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time(): connect: Final = _CapturingConnect(_SilentConnection()) - with patch("websockets.connect", connect): - assert await realtime_main._realtime_health_check( + with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised: + await realtime_main._realtime_health_check( model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01 ) + assert raised.value.status_code == 408 + assert "no server event within 0.01 seconds" in str(raised.value) @pytest.mark.asyncio From ef5f4712a961cff268222b84d4852d0287931c54 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:41:00 -0700 Subject: [PATCH 3/7] fix(health): treat invalid_model as healthy, read the first event only on OpenAI hosts, map rate limits to 429 --- litellm/llms/openai/realtime/handler.py | 18 ++++-------- litellm/realtime_api/main.py | 12 ++++---- litellm/responses/streaming_iterator.py | 1 + tests/unit/realtime_api/test_main.py | 37 +++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 18 deletions(-) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 3293da64082..3f1af2d3f37 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -14,7 +14,8 @@ from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES -from litellm.exceptions import AuthenticationError, BadRequestError, InternalServerError, Timeout +from litellm.exceptions import Timeout +from litellm.responses.streaming_iterator import map_stream_error_to_exception from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -33,6 +34,7 @@ if TYPE_CHECKING: from websockets.asyncio.client import ClientConnection _SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" def first_event_error(first_event: str | bytes) -> RealtimeError | None: @@ -42,16 +44,6 @@ def first_event_error(first_event: str | bytes) -> RealtimeError | None: return RealtimeErrorEvent.model_validate(event).error -def first_event_exception(error: RealtimeError, model: str) -> Exception: - match error: - case RealtimeError(code="invalid_api_key"): - return AuthenticationError(message=error.message, llm_provider="openai", model=model) - case RealtimeError(type="server_error"): - return InternalServerError(message=error.message, llm_provider="openai", model=model) - case _: - return BadRequestError(message=error.message, model=model, llm_provider="openai") - - async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]: try: first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) @@ -62,9 +54,9 @@ async def confirm_session_started(connection: "ClientConnection", model: str, ti llm_provider="openai", ) from None error: Final = first_event_error(first_event) - if error is None: + if error is None or error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: return True - raise first_event_exception(error, model) + raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") class OpenAIRealtime(OpenAIChatCompletion): diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 7fa67eb1698..1bc7db2655e 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -43,6 +43,7 @@ from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed from ..llms.custom_httpx.http_handler import realtime_ssl_for_url +from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async @@ -802,11 +803,12 @@ async def realtime_health_check( without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription Returns: - bool - True once the connection is open, and for OpenAI once the first server event is not an error, - since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports - the failure in its first server event + bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an + error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports + the failure in its first server event. An `invalid_model` error also counts as healthy: OpenAI checks + the key first, and Live and transcription models are served through other endpoints Raises: - Exception - if the connection is not successful, if OpenAI's first server event is an error, or if + Exception - if the connection is not successful, if OpenAI's first server event is any other error, or if OpenAI sends no server event within first_event_timeout_seconds """ import websockets @@ -905,7 +907,7 @@ async def realtime_health_check( max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=realtime_ssl_for_url(url), ) as connection: - if custom_llm_provider != "openai": + if custom_llm_provider != "openai" or not is_openai_backed_api_base(url): return True return await confirm_session_started(connection, model, first_event_timeout_seconds) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 4e418a91156..9ccc555348e 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -228,6 +228,7 @@ async def drain_logging_tasks(timeout: float = 5.0) -> None: _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( { "server_error": 500, + "invalid_api_key": 401, "rate_limit_exceeded": 429, "insufficient_quota": 429, "vector_store_timeout": 504, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index e2ce28f5585..70d62caa9dc 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -478,6 +478,18 @@ _OPENAI_SERVER_ERROR_EVENT: Final = ( '{"type": "error", "event_id": "event_3", ' '"error": {"type": "server_error", "code": null, "message": "The server had an error", "param": null}}' ) +_OPENAI_RATE_LIMIT_EVENT: Final = ( + '{"type": "error", "event_id": "event_5", "error": {"type": "tokens", "code": "rate_limit_exceeded", ' + '"message": "Rate limit reached for gpt-realtime", "param": null, "event_id": null}}' +) +_OPENAI_INSUFFICIENT_QUOTA_EVENT: Final = ( + '{"type": "error", "event_id": "event_6", "error": {"type": "insufficient_quota", "code": "insufficient_quota", ' + '"message": "You exceeded your current quota", "param": null, "event_id": null}}' +) +_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = ( + '{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", ' + '"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' +) _OPENAI_SESSION_CREATED_EVENT: Final = ( '{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}' ) @@ -704,6 +716,8 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch): (_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"), (_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"), (_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"), + (_OPENAI_RATE_LIMIT_EVENT, litellm.RateLimitError, 429, "Rate limit reached"), + (_OPENAI_INSUFFICIENT_QUOTA_EVENT, litellm.RateLimitError, 429, "You exceeded your current quota"), ], ) async def test_openai_health_check_reports_the_first_error_event_as_unhealthy( @@ -737,6 +751,29 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives(): ) +@pytest.mark.asyncio +async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted(): + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT)) + with patch("websockets.connect", connect): + assert await realtime_main.realtime_health_check( + model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real" + ) + + +@pytest.mark.asyncio +async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake(): + connect: Final = _CapturingConnect(_SilentConnection()) + with patch("websockets.connect", connect): + assert await realtime_main.realtime_health_check( + model="gpt-realtime", + custom_llm_provider="openai", + api_key="sk-real", + api_base="https://realtime-gateway.example/v1", + first_event_timeout_seconds=0.01, + ) + assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime" + + @pytest.mark.asyncio async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time(): connect: Final = _CapturingConnect(_SilentConnection()) From d8eaff8a0fa7f6b37c3edb64bf71760fd0660bc7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:05:40 -0700 Subject: [PATCH 4/7] fix(health): respect the global api_base, report a missing key as 401, keep invalid_model healthy only for known models Drop the separate first-event timeout so model_info.health_check_timeout governs the wait, and keep the shared stream error table unchanged by raising AuthenticationError for invalid_api_key in the realtime check --- litellm/constants.py | 1 - litellm/llms/openai/realtime/handler.py | 22 +++---- litellm/realtime_api/main.py | 23 ++++---- litellm/responses/streaming_iterator.py | 1 - tests/unit/realtime_api/test_main.py | 77 +++++++++++++++++++------ 5 files changed, 79 insertions(+), 45 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 098442c3ee9..1d5c6404bfd 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -416,7 +416,6 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0) -REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS: Final = 10.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index eab70e72b85..0d714a08b80 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,7 +4,6 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ -import asyncio import ssl from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, cast @@ -14,7 +13,7 @@ from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES -from litellm.exceptions import Timeout +from litellm.exceptions import AuthenticationError from litellm.responses.streaming_iterator import map_stream_error_to_exception from litellm.types.realtime import RealtimeQueryParams @@ -36,6 +35,7 @@ if TYPE_CHECKING: _SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" +_CREDENTIALS_REJECTED: Final = "invalid_api_key" def first_event_error(first_event: str | bytes) -> RealtimeError | None: @@ -45,18 +45,14 @@ def first_event_error(first_event: str | bytes) -> RealtimeError | None: return RealtimeErrorEvent.model_validate(event).error -async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]: - try: - first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) - except asyncio.TimeoutError: - raise Timeout( - message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake", - model=model, - llm_provider="openai", - ) from None - error: Final = first_event_error(first_event) - if error is None or error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: +async def confirm_session_started( + connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool +) -> Literal[True]: + error: Final = first_event_error(await connection.recv()) + if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known): return True + if error.code == _CREDENTIALS_REJECTED or not sent_credentials: + raise AuthenticationError(message=error.message, llm_provider="openai", model=model) raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 1bc7db2655e..4b92371b4b8 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -13,7 +13,6 @@ import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, - REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) @@ -32,7 +31,7 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -787,7 +786,6 @@ async def realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: Mapping[str, object] | None = None, - first_event_timeout_seconds: float = REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, ) -> bool: """ Health check for realtime API - tries connection to the realtime API websocket @@ -805,11 +803,10 @@ async def realtime_health_check( Returns: bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports - the failure in its first server event. An `invalid_model` error also counts as healthy: OpenAI checks - the key first, and Live and transcription models are served through other endpoints + the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as + healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints Raises: - Exception - if the connection is not successful, if OpenAI's first server event is any other error, or if - OpenAI sends no server event within first_event_timeout_seconds + Exception - if the connection is not successful or if OpenAI's first server event is any other error """ import websockets @@ -829,10 +826,11 @@ async def realtime_health_check( str | None, api_version or resolved_params.get("api_version") ) url: str | None = None + openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/" health_check_api_key: Final = ( await resolve_openai_bearer_token_async( static_api_key=openai_static_api_key(resolved_api_key), - api_base=resolved_api_base, + api_base=openai_api_base, litellm_params=resolved_params, ) if custom_llm_provider == LlmProviders.OPENAI.value @@ -857,10 +855,7 @@ async def realtime_health_check( query_params=azure_query_params, ) elif custom_llm_provider == "openai": - url = openai_realtime.construct_url( - api_base=resolved_api_base or "https://api.openai.com/", - query_params={"model": model}, - ) + url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model}) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model} @@ -909,7 +904,9 @@ async def realtime_health_check( ) as connection: if custom_llm_provider != "openai" or not is_openai_backed_api_base(url): return True - return await confirm_session_started(connection, model, first_event_timeout_seconds) + return await confirm_session_started( + connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None + ) _realtime_health_check = realtime_health_check diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 9ccc555348e..4e418a91156 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -228,7 +228,6 @@ async def drain_logging_tasks(timeout: float = 5.0) -> None: _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( { "server_error": 500, - "invalid_api_key": 401, "rate_limit_exceeded": 429, "insufficient_quota": 429, "vector_store_timeout": 504, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 70d62caa9dc..90d485145a4 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -490,6 +490,15 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = ( '{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", ' '"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' ) +_OPENAI_UNKNOWN_MODEL_EVENT: Final = ( + '{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", ' + '"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' +) +_OPENAI_MODEL_NOT_FOUND_EVENT: Final = ( + '{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", ' + '"message": "The model gpt-realtime-2.1x does not exist or you do not have access to it.", "param": null, ' + '"event_id": null}}' +) _OPENAI_SESSION_CREATED_EVENT: Final = ( '{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}' ) @@ -714,7 +723,6 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch): ("first_event", "expected_exception", "expected_status", "expected_message"), [ (_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"), - (_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"), (_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"), (_OPENAI_RATE_LIMIT_EVENT, litellm.RateLimitError, 429, "Rate limit reached"), (_OPENAI_INSUFFICIENT_QUOTA_EVENT, litellm.RateLimitError, 429, "You exceeded your current quota"), @@ -734,11 +742,12 @@ async def test_openai_health_check_reports_the_first_error_event_as_unhealthy( @pytest.mark.asyncio -async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_is_unhealthy(): +async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_fails_authentication(): connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_MISSING_AUTH_EVENT)) - with patch("websockets.connect", connect), pytest.raises(litellm.BadRequestError) as raised: + with patch("websockets.connect", connect), pytest.raises(litellm.AuthenticationError) as raised: await realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key=None) assert connect.kwargs["additional_headers"] == {} + assert raised.value.status_code == 401 assert "Missing bearer or basic authentication" in str(raised.value) @@ -752,7 +761,9 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives(): @pytest.mark.asyncio -async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted(): +async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted( + local_model_cost_map, +): connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT)) with patch("websockets.connect", connect): assert await realtime_main.realtime_health_check( @@ -761,28 +772,60 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim @pytest.mark.asyncio -async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake(): +@pytest.mark.parametrize( + ("model", "first_event", "expected_exception", "expected_message"), + [ + ( + "realtime-2.1", + _OPENAI_UNKNOWN_MODEL_EVENT, + litellm.BadRequestError, + 'Model "realtime-2.1" is not supported in realtime mode', + ), + ( + "gpt-realtime-2.1x", + _OPENAI_MODEL_NOT_FOUND_EVENT, + litellm.NotFoundError, + "The model gpt-realtime-2.1x does not exist", + ), + ], +) +async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_would_fail_on( + local_model_cost_map, model: str, first_event: str, expected_exception: type[Exception], expected_message: str +): + connect: Final = _CapturingConnect(_ScriptedConnection(first_event)) + with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised: + await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real") + assert expected_message in str(raised.value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_api_base", "global_api_base"), + [("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")], +) +async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake( + monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None +): + monkeypatch.setattr(litellm, "api_base", global_api_base) connect: Final = _CapturingConnect(_SilentConnection()) with patch("websockets.connect", connect): - assert await realtime_main.realtime_health_check( - model="gpt-realtime", - custom_llm_provider="openai", - api_key="sk-real", - api_base="https://realtime-gateway.example/v1", - first_event_timeout_seconds=0.01, + assert await asyncio.wait_for( + realtime_main.realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base + ), + timeout=5, ) assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime" @pytest.mark.asyncio -async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time(): +async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives(): connect: Final = _CapturingConnect(_SilentConnection()) - with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised: - await realtime_main.realtime_health_check( - model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01 + with patch("websockets.connect", connect), pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for( + realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real"), + timeout=0.05, ) - assert raised.value.status_code == 408 - assert "no server event within 0.01 seconds" in str(raised.value) @pytest.mark.asyncio From 42dff600f7281283f766053726d90ff401e31ae0 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:54:52 -0700 Subject: [PATCH 5/7] fix(health): keep invalid_model healthy unless the cost map lists the model outside realtime, and load realtime types lazily --- litellm/llms/openai/realtime/handler.py | 12 +++++++---- litellm/realtime_api/main.py | 27 +++++++++++++++++++++---- tests/unit/realtime_api/test_main.py | 17 ++++++++-------- tests/unit/test_lazy_imports.py | 2 +- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 0d714a08b80..79032974878 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -8,7 +8,6 @@ import ssl from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, cast -from openai.types.realtime import RealtimeError, RealtimeErrorEvent from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger @@ -31,6 +30,7 @@ from ..openai import OpenAIChatCompletion if TYPE_CHECKING: from fastapi import WebSocket + from openai.types.realtime import RealtimeError from websockets.asyncio.client import ClientConnection _SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) @@ -38,18 +38,22 @@ _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" _CREDENTIALS_REJECTED: Final = "invalid_api_key" -def first_event_error(first_event: str | bytes) -> RealtimeError | None: +def first_event_error(first_event: str | bytes) -> "RealtimeError | None": event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) if event.get("type") != "error": return None + from openai.types.realtime import RealtimeErrorEvent + return RealtimeErrorEvent.model_validate(event).error async def confirm_session_started( - connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool + connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool ) -> Literal[True]: error: Final = first_event_error(await connection.recv()) - if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known): + if error is None or ( + error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime + ): return True if error.code == _CREDENTIALS_REJECTED or not sent_credentials: raise AuthenticationError(message=error.message, llm_provider="openai", model=model) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 4b92371b4b8..4f12de61cd9 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -31,7 +31,12 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference +from litellm.utils import ( + ModelInfoLookupMiss, + ProviderConfigManager, + lookup_model_info_for_registration, + resolve_api_key_secret_reference, +) from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -741,6 +746,16 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) +_REALTIME_SERVED_MODES: Final = frozenset({"realtime", "audio_transcription"}) + + +def _cost_map_lists_outside_realtime(model: str) -> bool: + model_info: Final = lookup_model_info_for_registration(model, "openai") + if isinstance(model_info, ModelInfoLookupMiss) or model_info["mode"] in _REALTIME_SERVED_MODES: + return False + return not any(endpoint.startswith("/v1/realtime") for endpoint in model_info.get("supported_endpoints") or ()) + + _TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"} @@ -803,8 +818,9 @@ async def realtime_health_check( Returns: bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports - the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as - healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints + the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's + cost map lists the model under a mode realtime never serves: OpenAI checks the key first, and Live and + transcription models are served through other endpoints Raises: Exception - if the connection is not successful or if OpenAI's first server event is any other error """ @@ -905,7 +921,10 @@ async def realtime_health_check( if custom_llm_provider != "openai" or not is_openai_backed_api_base(url): return True return await confirm_session_started( - connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), ) diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 90d485145a4..789f51778f1 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -490,9 +490,9 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = ( '{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", ' '"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' ) -_OPENAI_UNKNOWN_MODEL_EVENT: Final = ( +_OPENAI_CHAT_MODEL_ON_REALTIME_EVENT: Final = ( '{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", ' - '"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' + '"message": "Model \\"gpt-5.5\\" is not supported in realtime mode.", "param": null, "event_id": null}}' ) _OPENAI_MODEL_NOT_FOUND_EVENT: Final = ( '{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", ' @@ -761,14 +761,13 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives(): @pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-live-1", "gpt-4o-transcribe", "gpt-audio", "gpt-live-unreleased"]) async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted( - local_model_cost_map, + local_model_cost_map, model: str ): connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT)) with patch("websockets.connect", connect): - assert await realtime_main.realtime_health_check( - model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real" - ) + assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real") @pytest.mark.asyncio @@ -776,10 +775,10 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim ("model", "first_event", "expected_exception", "expected_message"), [ ( - "realtime-2.1", - _OPENAI_UNKNOWN_MODEL_EVENT, + "gpt-5.5", + _OPENAI_CHAT_MODEL_ON_REALTIME_EVENT, litellm.BadRequestError, - 'Model "realtime-2.1" is not supported in realtime mode', + 'Model "gpt-5.5" is not supported in realtime mode', ), ( "gpt-realtime-2.1x", diff --git a/tests/unit/test_lazy_imports.py b/tests/unit/test_lazy_imports.py index 0416020a9b5..43aee4616d2 100644 --- a/tests/unit/test_lazy_imports.py +++ b/tests/unit/test_lazy_imports.py @@ -47,7 +47,7 @@ def test_import_litellm_does_not_load_fastapi_or_bpe_table(): [ sys.executable, "-c", - "import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding') if m in sys.modules))", + "import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding','openai.types.realtime') if m in sys.modules))", ], check=True, capture_output=True, From 69b2aa1339eb18482cff8be3c49ff86267fb7c25 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:34:27 -0700 Subject: [PATCH 6/7] fix(health): read the cost map's listed endpoints, not its mode, since registering a realtime deployment overwrites the mode --- litellm/realtime_api/main.py | 14 ++++++++------ tests/unit/realtime_api/test_main.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 4f12de61cd9..3fa7d9ae913 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -746,14 +746,15 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) -_REALTIME_SERVED_MODES: Final = frozenset({"realtime", "audio_transcription"}) +_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions") def _cost_map_lists_outside_realtime(model: str) -> bool: model_info: Final = lookup_model_info_for_registration(model, "openai") - if isinstance(model_info, ModelInfoLookupMiss) or model_info["mode"] in _REALTIME_SERVED_MODES: - return False - return not any(endpoint.startswith("/v1/realtime") for endpoint in model_info.get("supported_endpoints") or ()) + endpoints: Final = ( + () if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or () + ) + return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints) _TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"} @@ -819,8 +820,9 @@ async def realtime_health_check( bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's - cost map lists the model under a mode realtime never serves: OpenAI checks the key first, and Live and - transcription models are served through other endpoints + cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live + and transcription models are served through other endpoints. The listed `mode` is not read, since a + deployment's own `mode: realtime` replaces it when the router registers the deployment Raises: Exception - if the connection is not successful or if OpenAI's first server event is any other error """ diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 789f51778f1..881ecc1bd6e 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -504,6 +504,18 @@ _OPENAI_SESSION_CREATED_EVENT: Final = ( ) +def _register_as_realtime_deployment(model: str) -> None: + litellm.Router( + model_list=[ + { + "model_name": "realtime-deployment", + "litellm_params": {"model": f"openai/{model}", "api_key": "sk-real"}, + "model_info": {"mode": "realtime"}, + } + ] + ) + + class _CapturingConnect: def __init__(self, connection: _ServerEvents | None = None) -> None: self.url: str | None = None @@ -791,6 +803,7 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_would_fail_on( local_model_cost_map, model: str, first_event: str, expected_exception: type[Exception], expected_message: str ): + _register_as_realtime_deployment(model) connect: Final = _CapturingConnect(_ScriptedConnection(first_event)) with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised: await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real") From 393d9c63cafbc699ea36663cecf688be5dcc4148 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:55:30 -0700 Subject: [PATCH 7/7] fix(health): read a realtime gateway's first event and keep Live deployments listing /v1/live healthy Behind a custom api_base the OpenAI realtime health check now waits up to 3 seconds for the first event, so a gateway that passes on OpenAI's invalid_api_key error reports unhealthy while a silent gateway stays healthy. /v1/live counts as a realtime endpoint for the invalid_model rule. --- litellm/llms/openai/realtime/handler.py | 27 ++++++++++++-- litellm/realtime_api/main.py | 24 ++++++++++-- tests/unit/realtime_api/test_main.py | 49 +++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 79032974878..5be488f0f6e 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,6 +4,7 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, cast @@ -47,10 +48,10 @@ def first_event_error(first_event: str | bytes) -> "RealtimeError | None": return RealtimeErrorEvent.model_validate(event).error -async def confirm_session_started( - connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool +def _confirm_first_event( + first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool ) -> Literal[True]: - error: Final = first_event_error(await connection.recv()) + error: Final = first_event_error(first_event) if error is None or ( error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime ): @@ -60,6 +61,26 @@ async def confirm_session_started( raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") +async def confirm_session_started( + connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool +) -> Literal[True]: + return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime) + + +async def confirm_gateway_session_started( + connection: "ClientConnection", + model: str, + sent_credentials: bool, + model_listed_outside_realtime: bool, + first_event_wait_seconds: float, +) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds) + except asyncio.TimeoutError: + return True + return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime) + + class OpenAIRealtime(OpenAIChatCompletion): """ Base handler for OpenAI-compatible realtime WebSocket connections. diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 3fa7d9ae913..e9b534bdd89 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -49,7 +49,11 @@ from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_co from ..llms.custom_httpx.http_handler import realtime_ssl_for_url from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions -from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started +from ..llms.openai.realtime.handler import ( + OpenAIRealtime, + confirm_gateway_session_started, + confirm_session_started, +) from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config @@ -746,7 +750,8 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) -_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions") +_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live") +_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0 def _cost_map_lists_outside_realtime(model: str) -> bool: @@ -802,6 +807,7 @@ async def realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: Mapping[str, object] | None = None, + gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS, ) -> bool: """ Health check for realtime API - tries connection to the realtime API websocket @@ -819,7 +825,9 @@ async def realtime_health_check( Returns: bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports - the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's + the failure in its first server event. Behind any other OpenAI api_base the first event is read for + `gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy. + An `invalid_model` error also counts as healthy unless litellm's cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live and transcription models are served through other endpoints. The listed `mode` is not read, since a deployment's own `mode: realtime` replaces it when the router registers the deployment @@ -920,8 +928,16 @@ async def realtime_health_check( max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=realtime_ssl_for_url(url), ) as connection: - if custom_llm_provider != "openai" or not is_openai_backed_api_base(url): + if custom_llm_provider != "openai": return True + if not is_openai_backed_api_base(url): + return await confirm_gateway_session_started( + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), + first_event_wait_seconds=gateway_first_event_wait_seconds, + ) return await confirm_session_started( connection, model, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 881ecc1bd6e..5478b807691 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -504,13 +504,16 @@ _OPENAI_SESSION_CREATED_EVENT: Final = ( ) -def _register_as_realtime_deployment(model: str) -> None: +def _register_as_realtime_deployment(model: str, *supported_endpoints: str) -> None: litellm.Router( model_list=[ { "model_name": "realtime-deployment", "litellm_params": {"model": f"openai/{model}", "api_key": "sk-real"}, - "model_info": {"mode": "realtime"}, + "model_info": { + "mode": "realtime", + **({"supported_endpoints": list(supported_endpoints)} if supported_endpoints else {}), + }, } ] ) @@ -782,6 +785,18 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real") +@pytest.mark.asyncio +async def test_openai_health_check_is_healthy_for_a_live_deployment_that_lists_the_live_sessions_endpoint( + local_model_cost_map, +): + _register_as_realtime_deployment("gpt-live-1", "/v1/live/sessions") + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT)) + with patch("websockets.connect", connect): + assert await realtime_main.realtime_health_check( + model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real" + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( ("model", "first_event", "expected_exception", "expected_message"), @@ -815,7 +830,7 @@ async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_w ("deployment_api_base", "global_api_base"), [("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")], ) -async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake( +async def test_openai_health_check_through_a_compatible_gateway_is_healthy_when_it_sends_no_first_event( monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None ): monkeypatch.setattr(litellm, "api_base", global_api_base) @@ -823,13 +838,39 @@ async def test_openai_health_check_through_a_compatible_gateway_trusts_the_hands with patch("websockets.connect", connect): assert await asyncio.wait_for( realtime_main.realtime_health_check( - model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base + model="gpt-realtime", + custom_llm_provider="openai", + api_key="sk-real", + api_base=deployment_api_base, + gateway_first_event_wait_seconds=0.01, ), timeout=5, ) assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("connection", "expected_exception"), + [ + (_ScriptedConnection(_OPENAI_INVALID_API_KEY_EVENT), litellm.AuthenticationError), + (_ConnectionClosedBeforeAnyEvent(), ConnectionClosedError), + ], +) +async def test_openai_health_check_through_a_compatible_gateway_is_unhealthy_when_it_rejects_the_session( + connection: _ServerEvents, expected_exception: type[Exception] +): + connect: Final = _CapturingConnect(connection) + with patch("websockets.connect", connect), pytest.raises(expected_exception): + await realtime_main.realtime_health_check( + model="gpt-realtime", + custom_llm_provider="openai", + api_key="sk-wrong", + api_base="https://realtime-gateway.example/v1", + ) + assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime" + + @pytest.mark.asyncio async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives(): connect: Final = _CapturingConnect(_SilentConnection())