fix(health): keep invalid_model healthy unless the cost map lists the model outside realtime, and load realtime types lazily

This commit is contained in:
mateo-berri 2026-10-28 20:54:52 -07:00
parent d8eaff8a0f
commit 42dff600f7
4 changed files with 40 additions and 18 deletions

View file

@ -8,7 +8,6 @@ import ssl
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Literal, cast
from openai.types.realtime import RealtimeError, RealtimeErrorEvent
from pydantic import TypeAdapter
from litellm._logging import redact_string, verbose_logger
@ -31,6 +30,7 @@ from ..openai import OpenAIChatCompletion
if TYPE_CHECKING:
from fastapi import WebSocket
from openai.types.realtime import RealtimeError
from websockets.asyncio.client import ClientConnection
_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
@ -38,18 +38,22 @@ _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model"
_CREDENTIALS_REJECTED: Final = "invalid_api_key"
def first_event_error(first_event: str | bytes) -> RealtimeError | None:
def first_event_error(first_event: str | bytes) -> "RealtimeError | None":
event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event)
if event.get("type") != "error":
return None
from openai.types.realtime import RealtimeErrorEvent
return RealtimeErrorEvent.model_validate(event).error
async def confirm_session_started(
connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool
connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
) -> Literal[True]:
error: Final = first_event_error(await connection.recv())
if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known):
if error is None or (
error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime
):
return True
if error.code == _CREDENTIALS_REJECTED or not sent_credentials:
raise AuthenticationError(message=error.message, llm_provider="openai", model=model)

View file

@ -31,7 +31,12 @@ from litellm.types.realtime import (
)
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import CallTypes, LlmProviders
from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference
from litellm.utils import (
ModelInfoLookupMiss,
ProviderConfigManager,
lookup_model_info_for_registration,
resolve_api_key_secret_reference,
)
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
from ..litellm_core_utils.credential_accessor import CredentialAccessor
@ -741,6 +746,16 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
_REALTIME_SERVED_MODES: Final = frozenset({"realtime", "audio_transcription"})
def _cost_map_lists_outside_realtime(model: str) -> bool:
model_info: Final = lookup_model_info_for_registration(model, "openai")
if isinstance(model_info, ModelInfoLookupMiss) or model_info["mode"] in _REALTIME_SERVED_MODES:
return False
return not any(endpoint.startswith("/v1/realtime") for endpoint in model_info.get("supported_endpoints") or ())
_TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"}
@ -803,8 +818,9 @@ async def realtime_health_check(
Returns:
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as
healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints
the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's
cost map lists the model under a mode realtime never serves: OpenAI checks the key first, and Live and
transcription models are served through other endpoints
Raises:
Exception - if the connection is not successful or if OpenAI's first server event is any other error
"""
@ -905,7 +921,10 @@ async def realtime_health_check(
if custom_llm_provider != "openai" or not is_openai_backed_api_base(url):
return True
return await confirm_session_started(
connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None
connection,
model,
sent_credentials=bool(auth_headers),
model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
)

View file

@ -490,9 +490,9 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = (
'{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", '
'"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
)
_OPENAI_UNKNOWN_MODEL_EVENT: Final = (
_OPENAI_CHAT_MODEL_ON_REALTIME_EVENT: Final = (
'{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", '
'"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
'"message": "Model \\"gpt-5.5\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
)
_OPENAI_MODEL_NOT_FOUND_EVENT: Final = (
'{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", '
@ -761,14 +761,13 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives():
@pytest.mark.asyncio
@pytest.mark.parametrize("model", ["gpt-live-1", "gpt-4o-transcribe", "gpt-audio", "gpt-live-unreleased"])
async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted(
local_model_cost_map,
local_model_cost_map, model: str
):
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT))
with patch("websockets.connect", connect):
assert await realtime_main.realtime_health_check(
model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real"
)
assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")
@pytest.mark.asyncio
@ -776,10 +775,10 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
("model", "first_event", "expected_exception", "expected_message"),
[
(
"realtime-2.1",
_OPENAI_UNKNOWN_MODEL_EVENT,
"gpt-5.5",
_OPENAI_CHAT_MODEL_ON_REALTIME_EVENT,
litellm.BadRequestError,
'Model "realtime-2.1" is not supported in realtime mode',
'Model "gpt-5.5" is not supported in realtime mode',
),
(
"gpt-realtime-2.1x",

View file

@ -47,7 +47,7 @@ def test_import_litellm_does_not_load_fastapi_or_bpe_table():
[
sys.executable,
"-c",
"import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding') if m in sys.modules))",
"import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding','openai.types.realtime') if m in sys.modules))",
],
check=True,
capture_output=True,