fix(health): respect the global api_base, report a missing key as 401, keep invalid_model healthy only for known models

Drop the separate first-event timeout so model_info.health_check_timeout governs the wait, and keep the shared stream error table unchanged by raising AuthenticationError for invalid_api_key in the realtime check
This commit is contained in:
mateo-berri 2026-10-28 20:05:40 -07:00
parent 678104a8e5
commit d8eaff8a0f
5 changed files with 79 additions and 45 deletions

View file

@ -416,7 +416,6 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
)
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS: Final = 10.0
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123

View file

@ -4,7 +4,6 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
import asyncio
import ssl
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Literal, cast
@ -14,7 +13,7 @@ from pydantic import TypeAdapter
from litellm._logging import redact_string, verbose_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.exceptions import Timeout
from litellm.exceptions import AuthenticationError
from litellm.responses.streaming_iterator import map_stream_error_to_exception
from litellm.types.realtime import RealtimeQueryParams
@ -36,6 +35,7 @@ if TYPE_CHECKING:
_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model"
_CREDENTIALS_REJECTED: Final = "invalid_api_key"
def first_event_error(first_event: str | bytes) -> RealtimeError | None:
@ -45,18 +45,14 @@ def first_event_error(first_event: str | bytes) -> RealtimeError | None:
return RealtimeErrorEvent.model_validate(event).error
async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]:
try:
first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds)
except asyncio.TimeoutError:
raise Timeout(
message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake",
model=model,
llm_provider="openai",
) from None
error: Final = first_event_error(first_event)
if error is None or error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL:
async def confirm_session_started(
connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool
) -> Literal[True]:
error: Final = first_event_error(await connection.recv())
if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known):
return True
if error.code == _CREDENTIALS_REJECTED or not sent_credentials:
raise AuthenticationError(message=error.message, llm_provider="openai", model=model)
raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai")

View file

@ -13,7 +13,6 @@ import litellm
from litellm.constants import (
AZURE_OPENAI_AUDIO_PROVIDERS,
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS,
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
request_timeout,
)
@ -32,7 +31,7 @@ from litellm.types.realtime import (
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes, LlmProviders
from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference
from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
from ..litellm_core_utils.credential_accessor import CredentialAccessor
@ -787,7 +786,6 @@ async def realtime_health_check(
api_version: str | None = None,
realtime_protocol: str | None = None,
model_params: Mapping[str, object] | None = None,
first_event_timeout_seconds: float = REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS,
) -> bool:
"""
Health check for realtime API - tries connection to the realtime API websocket
@ -805,11 +803,10 @@ async def realtime_health_check(
Returns:
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
the failure in its first server event. An `invalid_model` error also counts as healthy: OpenAI checks
the key first, and Live and transcription models are served through other endpoints
the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as
healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints
Raises:
Exception - if the connection is not successful, if OpenAI's first server event is any other error, or if
OpenAI sends no server event within first_event_timeout_seconds
Exception - if the connection is not successful or if OpenAI's first server event is any other error
"""
import websockets
@ -829,10 +826,11 @@ async def realtime_health_check(
str | None, api_version or resolved_params.get("api_version")
)
url: str | None = None
openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/"
health_check_api_key: Final = (
await resolve_openai_bearer_token_async(
static_api_key=openai_static_api_key(resolved_api_key),
api_base=resolved_api_base,
api_base=openai_api_base,
litellm_params=resolved_params,
)
if custom_llm_provider == LlmProviders.OPENAI.value
@ -857,10 +855,7 @@ async def realtime_health_check(
query_params=azure_query_params,
)
elif custom_llm_provider == "openai":
url = openai_realtime.construct_url(
api_base=resolved_api_base or "https://api.openai.com/",
query_params={"model": model},
)
url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model})
elif custom_llm_provider == "xai":
url = xai_realtime.construct_url(
api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model}
@ -909,7 +904,9 @@ async def realtime_health_check(
) as connection:
if custom_llm_provider != "openai" or not is_openai_backed_api_base(url):
return True
return await confirm_session_started(connection, model, first_event_timeout_seconds)
return await confirm_session_started(
connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None
)
_realtime_health_check = realtime_health_check

View file

@ -228,7 +228,6 @@ async def drain_logging_tasks(timeout: float = 5.0) -> None:
_ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType(
{
"server_error": 500,
"invalid_api_key": 401,
"rate_limit_exceeded": 429,
"insufficient_quota": 429,
"vector_store_timeout": 504,

View file

@ -490,6 +490,15 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = (
'{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", '
'"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
)
_OPENAI_UNKNOWN_MODEL_EVENT: Final = (
'{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", '
'"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
)
_OPENAI_MODEL_NOT_FOUND_EVENT: Final = (
'{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", '
'"message": "The model gpt-realtime-2.1x does not exist or you do not have access to it.", "param": null, '
'"event_id": null}}'
)
_OPENAI_SESSION_CREATED_EVENT: Final = (
'{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}'
)
@ -714,7 +723,6 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch):
("first_event", "expected_exception", "expected_status", "expected_message"),
[
(_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"),
(_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"),
(_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"),
(_OPENAI_RATE_LIMIT_EVENT, litellm.RateLimitError, 429, "Rate limit reached"),
(_OPENAI_INSUFFICIENT_QUOTA_EVENT, litellm.RateLimitError, 429, "You exceeded your current quota"),
@ -734,11 +742,12 @@ async def test_openai_health_check_reports_the_first_error_event_as_unhealthy(
@pytest.mark.asyncio
async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_is_unhealthy():
async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_fails_authentication():
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_MISSING_AUTH_EVENT))
with patch("websockets.connect", connect), pytest.raises(litellm.BadRequestError) as raised:
with patch("websockets.connect", connect), pytest.raises(litellm.AuthenticationError) as raised:
await realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key=None)
assert connect.kwargs["additional_headers"] == {}
assert raised.value.status_code == 401
assert "Missing bearer or basic authentication" in str(raised.value)
@ -752,7 +761,9 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives():
@pytest.mark.asyncio
async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted():
async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted(
local_model_cost_map,
):
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT))
with patch("websockets.connect", connect):
assert await realtime_main.realtime_health_check(
@ -761,28 +772,60 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
@pytest.mark.asyncio
async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake():
@pytest.mark.parametrize(
("model", "first_event", "expected_exception", "expected_message"),
[
(
"realtime-2.1",
_OPENAI_UNKNOWN_MODEL_EVENT,
litellm.BadRequestError,
'Model "realtime-2.1" is not supported in realtime mode',
),
(
"gpt-realtime-2.1x",
_OPENAI_MODEL_NOT_FOUND_EVENT,
litellm.NotFoundError,
"The model gpt-realtime-2.1x does not exist",
),
],
)
async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_would_fail_on(
local_model_cost_map, model: str, first_event: str, expected_exception: type[Exception], expected_message: str
):
connect: Final = _CapturingConnect(_ScriptedConnection(first_event))
with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised:
await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")
assert expected_message in str(raised.value)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_api_base", "global_api_base"),
[("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")],
)
async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake(
monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None
):
monkeypatch.setattr(litellm, "api_base", global_api_base)
connect: Final = _CapturingConnect(_SilentConnection())
with patch("websockets.connect", connect):
assert await realtime_main.realtime_health_check(
model="gpt-realtime",
custom_llm_provider="openai",
api_key="sk-real",
api_base="https://realtime-gateway.example/v1",
first_event_timeout_seconds=0.01,
assert await asyncio.wait_for(
realtime_main.realtime_health_check(
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base
),
timeout=5,
)
assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime"
@pytest.mark.asyncio
async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time():
async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives():
connect: Final = _CapturingConnect(_SilentConnection())
with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised:
await realtime_main.realtime_health_check(
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01
with patch("websockets.connect", connect), pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(
realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real"),
timeout=0.05,
)
assert raised.value.status_code == 408
assert "no server event within 0.01 seconds" in str(raised.value)
@pytest.mark.asyncio