From a7c2be768542e6ecdd338f8c41278a11bc578935 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:39:03 -0700 Subject: [PATCH] fix(health): time out a silent openai realtime handshake and keep the first-event parsing in the openai handler --- litellm/llms/openai/realtime/handler.py | 45 +++++++++++++++++++- litellm/realtime_api/main.py | 44 ++----------------- tests/test_litellm/realtime_api/test_main.py | 8 ++-- 3 files changed, 53 insertions(+), 44 deletions(-) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index e3ecbac1a53..cdac99721e5 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl -from typing import Any, Final, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, Literal, cast + +from openai.types.realtime import RealtimeError, RealtimeErrorEvent +from pydantic import TypeAdapter from litellm._logging import _redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.exceptions import AuthenticationError, BadRequestError, InternalServerError, Timeout from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -20,6 +26,43 @@ from ....litellm_core_utils.realtime_streaming import ( from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..openai import OpenAIChatCompletion +if TYPE_CHECKING: + from websockets.asyncio.client import ClientConnection + +_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def first_event_error(first_event: str | bytes) -> RealtimeError | None: + event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) + if event.get("type") != "error": + return None + return RealtimeErrorEvent.model_validate(event).error + + +def first_event_exception(error: RealtimeError, model: str) -> Exception: + match error: + case RealtimeError(code="invalid_api_key"): + return AuthenticationError(message=error.message, llm_provider="openai", model=model) + case RealtimeError(type="server_error"): + return InternalServerError(message=error.message, llm_provider="openai", model=model) + case _: + return BadRequestError(message=error.message, model=model, llm_provider="openai") + + +async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) + except asyncio.TimeoutError: + raise Timeout( + message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake", + model=model, + llm_provider="openai", + ) from None + error: Final = first_event_error(first_event) + if error is None: + return True + raise first_event_exception(error, model) + class OpenAIRealtime(OpenAIChatCompletion): """ diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index c24c34e19c9..1b87c7a1eea 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -6,9 +6,6 @@ from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, cast -from openai.types.realtime import RealtimeError, RealtimeErrorEvent -from pydantic import TypeAdapter - import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, @@ -41,7 +38,7 @@ from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime from ..llms.custom_httpx.http_handler import get_shared_realtime_ssl_context -from ..llms.openai.realtime.handler import OpenAIRealtime +from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig from ..llms.vertex_ai.vertex_llm_base import VertexBase from ..llms.xai.realtime.handler import XAIRealtime @@ -49,7 +46,6 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket - from websockets.asyncio.client import ClientConnection from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig @@ -619,39 +615,6 @@ def _realtime_health_check_auth_headers( return MappingProxyType({"Authorization": f"Bearer {api_key}"}) -_REALTIME_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) - - -def _realtime_first_event_error(first_event: str | bytes) -> RealtimeError | None: - event: Final = _REALTIME_SERVER_EVENT_FIELDS.validate_json(first_event) - if event.get("type") != "error": - return None - return RealtimeErrorEvent.model_validate(event).error - - -def _realtime_first_event_exception(error: RealtimeError, model: str) -> Exception: - match error: - case RealtimeError(code="invalid_api_key"): - return litellm.AuthenticationError(message=error.message, llm_provider="openai", model=model) - case RealtimeError(type="server_error"): - return litellm.InternalServerError(message=error.message, llm_provider="openai", model=model) - case _: - return litellm.BadRequestError(message=error.message, model=model, llm_provider="openai") - - -async def _confirm_realtime_session_started( - connection: "ClientConnection", model: str, timeout_seconds: float -) -> Literal[True]: - try: - first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) - except asyncio.TimeoutError: - return True - error: Final = _realtime_first_event_error(first_event) - if error is None: - return True - raise _realtime_first_event_exception(error, model) - - async def _realtime_health_check( model: str, custom_llm_provider: str, @@ -680,7 +643,8 @@ async def _realtime_health_check( since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports the failure in its first server event Raises: - Exception - if the connection is not successful, or if OpenAI's first server event is an error + Exception - if the connection is not successful, if OpenAI's first server event is an error, or if + OpenAI sends no server event within first_event_timeout_seconds """ import websockets @@ -763,4 +727,4 @@ async def _realtime_health_check( ) as connection: if custom_llm_provider != "openai": return True - return await _confirm_realtime_session_started(connection, model, first_event_timeout_seconds) + return await confirm_session_started(connection, model, first_event_timeout_seconds) diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index 8213e2bdf3e..911f9c74891 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -493,12 +493,14 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives(): @pytest.mark.asyncio -async def test_openai_health_check_stays_healthy_when_no_first_event_arrives_in_time(): +async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time(): connect: Final = _CapturingConnect(_SilentConnection()) - with patch("websockets.connect", connect): - assert await realtime_main._realtime_health_check( + with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised: + await realtime_main._realtime_health_check( model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01 ) + assert raised.value.status_code == 408 + assert "no server event within 0.01 seconds" in str(raised.value) @pytest.mark.asyncio