fix(health): read a realtime gateway's first event and keep Live deployments listing /v1/live healthy
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Behind a custom api_base the OpenAI realtime health check now waits up to
3 seconds for the first event, so a gateway that passes on OpenAI's
invalid_api_key error reports unhealthy while a silent gateway stays
healthy. /v1/live counts as a realtime endpoint for the invalid_model rule.
This commit is contained in:
mateo-berri 2026-10-28 22:55:30 -07:00
parent 69b2aa1339
commit 393d9c63ca
3 changed files with 89 additions and 11 deletions

View file

@ -4,6 +4,7 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
import asyncio
import ssl
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Literal, cast
@ -47,10 +48,10 @@ def first_event_error(first_event: str | bytes) -> "RealtimeError | None":
return RealtimeErrorEvent.model_validate(event).error
async def confirm_session_started(
connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
def _confirm_first_event(
first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool
) -> Literal[True]:
error: Final = first_event_error(await connection.recv())
error: Final = first_event_error(first_event)
if error is None or (
error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime
):
@ -60,6 +61,26 @@ async def confirm_session_started(
raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai")
async def confirm_session_started(
connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
) -> Literal[True]:
return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime)
async def confirm_gateway_session_started(
connection: "ClientConnection",
model: str,
sent_credentials: bool,
model_listed_outside_realtime: bool,
first_event_wait_seconds: float,
) -> Literal[True]:
try:
first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds)
except asyncio.TimeoutError:
return True
return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime)
class OpenAIRealtime(OpenAIChatCompletion):
"""
Base handler for OpenAI-compatible realtime WebSocket connections.

View file

@ -49,7 +49,11 @@ from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_co
from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
from ..llms.openai.common_utils import is_openai_backed_api_base
from ..llms.openai.live.handler import OpenAILiveSessions
from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started
from ..llms.openai.realtime.handler import (
OpenAIRealtime,
confirm_gateway_session_started,
confirm_session_started,
)
from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async
from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model
from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config
@ -746,7 +750,8 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions")
_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live")
_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0
def _cost_map_lists_outside_realtime(model: str) -> bool:
@ -802,6 +807,7 @@ async def realtime_health_check(
api_version: str | None = None,
realtime_protocol: str | None = None,
model_params: Mapping[str, object] | None = None,
gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS,
) -> bool:
"""
Health check for realtime API - tries connection to the realtime API websocket
@ -819,7 +825,9 @@ async def realtime_health_check(
Returns:
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's
the failure in its first server event. Behind any other OpenAI api_base the first event is read for
`gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy.
An `invalid_model` error also counts as healthy unless litellm's
cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live
and transcription models are served through other endpoints. The listed `mode` is not read, since a
deployment's own `mode: realtime` replaces it when the router registers the deployment
@ -920,8 +928,16 @@ async def realtime_health_check(
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=realtime_ssl_for_url(url),
) as connection:
if custom_llm_provider != "openai" or not is_openai_backed_api_base(url):
if custom_llm_provider != "openai":
return True
if not is_openai_backed_api_base(url):
return await confirm_gateway_session_started(
connection,
model,
sent_credentials=bool(auth_headers),
model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
first_event_wait_seconds=gateway_first_event_wait_seconds,
)
return await confirm_session_started(
connection,
model,

View file

@ -504,13 +504,16 @@ _OPENAI_SESSION_CREATED_EVENT: Final = (
)
def _register_as_realtime_deployment(model: str) -> None:
def _register_as_realtime_deployment(model: str, *supported_endpoints: str) -> None:
litellm.Router(
model_list=[
{
"model_name": "realtime-deployment",
"litellm_params": {"model": f"openai/{model}", "api_key": "sk-real"},
"model_info": {"mode": "realtime"},
"model_info": {
"mode": "realtime",
**({"supported_endpoints": list(supported_endpoints)} if supported_endpoints else {}),
},
}
]
)
@ -782,6 +785,18 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")
@pytest.mark.asyncio
async def test_openai_health_check_is_healthy_for_a_live_deployment_that_lists_the_live_sessions_endpoint(
local_model_cost_map,
):
_register_as_realtime_deployment("gpt-live-1", "/v1/live/sessions")
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT))
with patch("websockets.connect", connect):
assert await realtime_main.realtime_health_check(
model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model", "first_event", "expected_exception", "expected_message"),
@ -815,7 +830,7 @@ async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_w
("deployment_api_base", "global_api_base"),
[("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")],
)
async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake(
async def test_openai_health_check_through_a_compatible_gateway_is_healthy_when_it_sends_no_first_event(
monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None
):
monkeypatch.setattr(litellm, "api_base", global_api_base)
@ -823,13 +838,39 @@ async def test_openai_health_check_through_a_compatible_gateway_trusts_the_hands
with patch("websockets.connect", connect):
assert await asyncio.wait_for(
realtime_main.realtime_health_check(
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base
model="gpt-realtime",
custom_llm_provider="openai",
api_key="sk-real",
api_base=deployment_api_base,
gateway_first_event_wait_seconds=0.01,
),
timeout=5,
)
assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("connection", "expected_exception"),
[
(_ScriptedConnection(_OPENAI_INVALID_API_KEY_EVENT), litellm.AuthenticationError),
(_ConnectionClosedBeforeAnyEvent(), ConnectionClosedError),
],
)
async def test_openai_health_check_through_a_compatible_gateway_is_unhealthy_when_it_rejects_the_session(
connection: _ServerEvents, expected_exception: type[Exception]
):
connect: Final = _CapturingConnect(connection)
with patch("websockets.connect", connect), pytest.raises(expected_exception):
await realtime_main.realtime_health_check(
model="gpt-realtime",
custom_llm_provider="openai",
api_key="sk-wrong",
api_base="https://realtime-gateway.example/v1",
)
assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime"
@pytest.mark.asyncio
async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives():
connect: Final = _CapturingConnect(_SilentConnection())