From d8eaff8a0fa7f6b37c3edb64bf71760fd0660bc7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:05:40 -0700 Subject: [PATCH] fix(health): respect the global api_base, report a missing key as 401, keep invalid_model healthy only for known models Drop the separate first-event timeout so model_info.health_check_timeout governs the wait, and keep the shared stream error table unchanged by raising AuthenticationError for invalid_api_key in the realtime check --- litellm/constants.py | 1 - litellm/llms/openai/realtime/handler.py | 22 +++---- litellm/realtime_api/main.py | 23 ++++---- litellm/responses/streaming_iterator.py | 1 - tests/unit/realtime_api/test_main.py | 77 +++++++++++++++++++------ 5 files changed, 79 insertions(+), 45 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 098442c3ee9..1d5c6404bfd 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -416,7 +416,6 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0) -REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS: Final = 10.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index eab70e72b85..0d714a08b80 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -4,7 +4,6 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint. This requires websockets, and is currently only supported on LiteLLM Proxy. """ -import asyncio import ssl from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, cast @@ -14,7 +13,7 @@ from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES -from litellm.exceptions import Timeout +from litellm.exceptions import AuthenticationError from litellm.responses.streaming_iterator import map_stream_error_to_exception from litellm.types.realtime import RealtimeQueryParams @@ -36,6 +35,7 @@ if TYPE_CHECKING: _SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" +_CREDENTIALS_REJECTED: Final = "invalid_api_key" def first_event_error(first_event: str | bytes) -> RealtimeError | None: @@ -45,18 +45,14 @@ def first_event_error(first_event: str | bytes) -> RealtimeError | None: return RealtimeErrorEvent.model_validate(event).error -async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]: - try: - first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds) - except asyncio.TimeoutError: - raise Timeout( - message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake", - model=model, - llm_provider="openai", - ) from None - error: Final = first_event_error(first_event) - if error is None or error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: +async def confirm_session_started( + connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool +) -> Literal[True]: + error: Final = first_event_error(await connection.recv()) + if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known): return True + if error.code == _CREDENTIALS_REJECTED or not sent_credentials: + raise AuthenticationError(message=error.message, llm_provider="openai", model=model) raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai") diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 1bc7db2655e..4b92371b4b8 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -13,7 +13,6 @@ import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, - REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) @@ -32,7 +31,7 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -787,7 +786,6 @@ async def realtime_health_check( api_version: str | None = None, realtime_protocol: str | None = None, model_params: Mapping[str, object] | None = None, - first_event_timeout_seconds: float = REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS, ) -> bool: """ Health check for realtime API - tries connection to the realtime API websocket @@ -805,11 +803,10 @@ async def realtime_health_check( Returns: bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports - the failure in its first server event. An `invalid_model` error also counts as healthy: OpenAI checks - the key first, and Live and transcription models are served through other endpoints + the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as + healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints Raises: - Exception - if the connection is not successful, if OpenAI's first server event is any other error, or if - OpenAI sends no server event within first_event_timeout_seconds + Exception - if the connection is not successful or if OpenAI's first server event is any other error """ import websockets @@ -829,10 +826,11 @@ async def realtime_health_check( str | None, api_version or resolved_params.get("api_version") ) url: str | None = None + openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/" health_check_api_key: Final = ( await resolve_openai_bearer_token_async( static_api_key=openai_static_api_key(resolved_api_key), - api_base=resolved_api_base, + api_base=openai_api_base, litellm_params=resolved_params, ) if custom_llm_provider == LlmProviders.OPENAI.value @@ -857,10 +855,7 @@ async def realtime_health_check( query_params=azure_query_params, ) elif custom_llm_provider == "openai": - url = openai_realtime.construct_url( - api_base=resolved_api_base or "https://api.openai.com/", - query_params={"model": model}, - ) + url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model}) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model} @@ -909,7 +904,9 @@ async def realtime_health_check( ) as connection: if custom_llm_provider != "openai" or not is_openai_backed_api_base(url): return True - return await confirm_session_started(connection, model, first_event_timeout_seconds) + return await confirm_session_started( + connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None + ) _realtime_health_check = realtime_health_check diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 9ccc555348e..4e418a91156 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -228,7 +228,6 @@ async def drain_logging_tasks(timeout: float = 5.0) -> None: _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( { "server_error": 500, - "invalid_api_key": 401, "rate_limit_exceeded": 429, "insufficient_quota": 429, "vector_store_timeout": 504, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 70d62caa9dc..90d485145a4 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -490,6 +490,15 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = ( '{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", ' '"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' ) +_OPENAI_UNKNOWN_MODEL_EVENT: Final = ( + '{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", ' + '"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' +) +_OPENAI_MODEL_NOT_FOUND_EVENT: Final = ( + '{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", ' + '"message": "The model gpt-realtime-2.1x does not exist or you do not have access to it.", "param": null, ' + '"event_id": null}}' +) _OPENAI_SESSION_CREATED_EVENT: Final = ( '{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}' ) @@ -714,7 +723,6 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch): ("first_event", "expected_exception", "expected_status", "expected_message"), [ (_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"), - (_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"), (_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"), (_OPENAI_RATE_LIMIT_EVENT, litellm.RateLimitError, 429, "Rate limit reached"), (_OPENAI_INSUFFICIENT_QUOTA_EVENT, litellm.RateLimitError, 429, "You exceeded your current quota"), @@ -734,11 +742,12 @@ async def test_openai_health_check_reports_the_first_error_event_as_unhealthy( @pytest.mark.asyncio -async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_is_unhealthy(): +async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_fails_authentication(): connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_MISSING_AUTH_EVENT)) - with patch("websockets.connect", connect), pytest.raises(litellm.BadRequestError) as raised: + with patch("websockets.connect", connect), pytest.raises(litellm.AuthenticationError) as raised: await realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key=None) assert connect.kwargs["additional_headers"] == {} + assert raised.value.status_code == 401 assert "Missing bearer or basic authentication" in str(raised.value) @@ -752,7 +761,9 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives(): @pytest.mark.asyncio -async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted(): +async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted( + local_model_cost_map, +): connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT)) with patch("websockets.connect", connect): assert await realtime_main.realtime_health_check( @@ -761,28 +772,60 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim @pytest.mark.asyncio -async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake(): +@pytest.mark.parametrize( + ("model", "first_event", "expected_exception", "expected_message"), + [ + ( + "realtime-2.1", + _OPENAI_UNKNOWN_MODEL_EVENT, + litellm.BadRequestError, + 'Model "realtime-2.1" is not supported in realtime mode', + ), + ( + "gpt-realtime-2.1x", + _OPENAI_MODEL_NOT_FOUND_EVENT, + litellm.NotFoundError, + "The model gpt-realtime-2.1x does not exist", + ), + ], +) +async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_would_fail_on( + local_model_cost_map, model: str, first_event: str, expected_exception: type[Exception], expected_message: str +): + connect: Final = _CapturingConnect(_ScriptedConnection(first_event)) + with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised: + await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real") + assert expected_message in str(raised.value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_api_base", "global_api_base"), + [("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")], +) +async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake( + monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None +): + monkeypatch.setattr(litellm, "api_base", global_api_base) connect: Final = _CapturingConnect(_SilentConnection()) with patch("websockets.connect", connect): - assert await realtime_main.realtime_health_check( - model="gpt-realtime", - custom_llm_provider="openai", - api_key="sk-real", - api_base="https://realtime-gateway.example/v1", - first_event_timeout_seconds=0.01, + assert await asyncio.wait_for( + realtime_main.realtime_health_check( + model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base + ), + timeout=5, ) assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime" @pytest.mark.asyncio -async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time(): +async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives(): connect: Final = _CapturingConnect(_SilentConnection()) - with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised: - await realtime_main.realtime_health_check( - model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01 + with patch("websockets.connect", connect), pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for( + realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real"), + timeout=0.05, ) - assert raised.value.status_code == 408 - assert "no server event within 0.01 seconds" in str(raised.value) @pytest.mark.asyncio