diff --git a/litellm/constants.py b/litellm/constants.py index d85e104ba15..65b63f2355b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -313,6 +313,7 @@ REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = int(_max_size_env) if _max_si REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float( os.getenv("REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", "20.0") ) +REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS: Final = 10.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index e3ecbac1a53..cdac99721e5 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ +import asyncio import ssl -from typing import Any, Final, cast +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, Final, Literal, cast + +from openai.types.realtime import RealtimeError, RealtimeErrorEvent +from pydantic import TypeAdapter from litellm._logging import _redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.exceptions import AuthenticationError, BadRequestError, InternalServerError, Timeout from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -20,6 +26,43 @@ from ....litellm_core_utils.realtime_streaming import ( from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from ..openai import OpenAIChatCompletion +if TYPE_CHECKING: + from websockets.asyncio.client import ClientConnection + +_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def first_event_error(first_event: str | bytes) -> RealtimeError | None: + event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) + if event.get("type") != "error": + return None + return RealtimeErrorEvent.model_validate(event).error + + +def first_event_exception(error: RealtimeError, model: str) -> Exception: + match error: + case RealtimeError(code="invalid_api_key"): + return AuthenticationError(message=error.message, llm_provider="openai", model=model) + case RealtimeError(type="server_error"): + return InternalServerError(message=error.message, llm_provider="openai", model=model) + case _: + return BadRequestError(message=error.message, model=model, llm_provider="openai") + + +async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]: + try: + first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) + except asyncio.TimeoutError: + raise Timeout( + message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake", + model=model, + llm_provider="openai", + ) from None + error: Final = first_event_error(first_event) + if error is None: + return True + raise first_event_exception(error, model) + class OpenAIRealtime(OpenAIChatCompletion): """ diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index d70295e9a2a..191af9cbf6c 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -10,6 +10,7 @@ import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, + REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) @@ -37,7 +38,7 @@ from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime from ..llms.custom_httpx.http_handler import get_shared_realtime_ssl_context -from ..llms.openai.realtime.handler import OpenAIRealtime +from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig from ..llms.vertex_ai.vertex_llm_base import VertexBase from ..llms.xai.realtime.handler import XAIRealtime @@ -624,6 +625,7 @@ async def _realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: dict | None = None, + first_event_timeout_seconds: float = REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, ): """ Health check for realtime API - tries connection to the realtime API websocket @@ -639,9 +641,12 @@ async def _realtime_health_check( without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription Returns: - bool - True if connection is successful, False otherwise + bool - True once the connection is open, and for OpenAI once the first server event is not an error, + since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports + the failure in its first server event Raises: - Exception - if the connection is not successful + Exception - if the connection is not successful, if OpenAI's first server event is an error, or if + OpenAI sends no server event within first_event_timeout_seconds """ import websockets @@ -721,5 +726,7 @@ async def _realtime_health_check( additional_headers=auth_headers, max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ssl=ssl_context, - ): - return True + ) as connection: + if custom_llm_provider != "openai": + return True + return await confirm_session_started(connection, model, first_event_timeout_seconds) diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py index 643e65af47c..911f9c74891 100644 --- a/tests/test_litellm/realtime_api/test_main.py +++ b/tests/test_litellm/realtime_api/test_main.py @@ -5,6 +5,8 @@ from typing import Final from unittest.mock import MagicMock, patch import pytest +from websockets.exceptions import ConnectionClosedError +from websockets.frames import Close import litellm from litellm.models.credentials import CredentialItem @@ -223,17 +225,18 @@ def test_client_secret_forwards_nested_transcription_model_untouched(monkeypatch class _CapturingConnect: - def __init__(self) -> None: + def __init__(self, connection: object | None = None) -> None: self.url: str | None = None self.kwargs: dict[str, object] = {} + self._connection: Final = connection if connection is not None else MagicMock() def __call__(self, url: str, **kwargs: object) -> "_CapturingConnect": self.url = url self.kwargs = kwargs return self - async def __aenter__(self) -> MagicMock: - return MagicMock() + async def __aenter__(self) -> object: + return self._connection async def __aexit__( self, @@ -298,7 +301,7 @@ async def test_bearer_health_check_sends_stored_credential_as_bearer_token( ) ], ) - connect = _CapturingConnect() + connect = _CapturingConnect(_ScriptedConnection(_OPENAI_SESSION_CREATED_EVENT)) with patch("websockets.connect", connect): assert await realtime_main._realtime_health_check( model=model, @@ -412,6 +415,113 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch): assert connect.url == _AZURE_BETA_HEALTH_URL +class _ScriptedConnection: + def __init__(self, *frames: str) -> None: + self._frames: Final = iter(frames) + + async def recv(self) -> str: + return next(self._frames) + + +class _SilentConnection: + async def recv(self) -> str: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +class _ConnectionClosedBeforeAnyEvent: + async def recv(self) -> str: + raise ConnectionClosedError(Close(3000, "invalid_api_key"), None) + + +_OPENAI_INVALID_API_KEY_EVENT: Final = ( + '{"type": "error", "event_id": "event_1", "error": {"type": "invalid_request_error", "code": "invalid_api_key", ' + '"message": "Incorrect API key provided: sk-proj-****0000. You can find your API key at ' + 'https://platform.openai.com/account/api-keys.", "param": null, "event_id": null}}' +) +_OPENAI_MISSING_AUTH_EVENT: Final = ( + '{"type": "error", "event_id": "event_2", "error": {"type": "invalid_request_error", "code": null, ' + '"message": "Missing bearer or basic authentication in header", "param": null, "event_id": null}}' +) +_OPENAI_SERVER_ERROR_EVENT: Final = ( + '{"type": "error", "event_id": "event_3", ' + '"error": {"type": "server_error", "code": null, "message": "The server had an error", "param": null}}' +) +_OPENAI_SESSION_CREATED_EVENT: Final = ( + '{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}' +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("first_event", "expected_exception", "expected_status", "expected_message"), + [ + (_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"), + (_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"), + (_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"), + ], +) +async def test_openai_health_check_reports_the_first_error_event_as_unhealthy( + first_event: str, expected_exception: type[Exception], expected_status: int, expected_message: str +): + connect: Final = _CapturingConnect(_ScriptedConnection(first_event)) + with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised: + await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-wrong" + ) + assert raised.value.status_code == expected_status + assert expected_message in str(raised.value) + assert connect.url == "wss://api.openai.com/v1/realtime?model=gpt-realtime" + + +@pytest.mark.asyncio +async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_is_unhealthy(): + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_MISSING_AUTH_EVENT)) + with patch("websockets.connect", connect), pytest.raises(litellm.BadRequestError) as raised: + await realtime_main._realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key=None) + assert connect.kwargs["additional_headers"] == {} + assert "Missing bearer or basic authentication" in str(raised.value) + + +@pytest.mark.asyncio +async def test_openai_health_check_is_healthy_once_session_created_arrives(): + connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_SESSION_CREATED_EVENT)) + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real" + ) + + +@pytest.mark.asyncio +async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time(): + connect: Final = _CapturingConnect(_SilentConnection()) + with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised: + await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01 + ) + assert raised.value.status_code == 408 + assert "no server event within 0.01 seconds" in str(raised.value) + + +@pytest.mark.asyncio +async def test_openai_health_check_is_unhealthy_when_the_socket_closes_before_any_event(): + connect: Final = _CapturingConnect(_ConnectionClosedBeforeAnyEvent()) + with patch("websockets.connect", connect), pytest.raises(ConnectionClosedError): + await realtime_main._realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-wrong" + ) + + +@pytest.mark.asyncio +async def test_xai_health_check_trusts_the_handshake_without_reading_a_first_event(): + connect: Final = _CapturingConnect(_ScriptedConnection()) + with patch("websockets.connect", connect): + assert await realtime_main._realtime_health_check( + model="grok-voice-latest", custom_llm_provider="xai", api_key="sk-wrong" + ) + assert connect.url == "wss://api.x.ai/v1/realtime?model=grok-voice-latest" + + class _ConnectThatStopsAfterCapturingTheUrl: url: str | None = None