fix(health): read the cost map's listed endpoints, not its mode, since registering a realtime deployment overwrites the mode

This commit is contained in:
mateo-berri 2026-10-28 21:34:27 -07:00
parent 42dff600f7
commit 69b2aa1339
2 changed files with 21 additions and 6 deletions

View file

@ -746,14 +746,15 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
_REALTIME_SERVED_MODES: Final = frozenset({"realtime", "audio_transcription"})
_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions")
def _cost_map_lists_outside_realtime(model: str) -> bool:
model_info: Final = lookup_model_info_for_registration(model, "openai")
if isinstance(model_info, ModelInfoLookupMiss) or model_info["mode"] in _REALTIME_SERVED_MODES:
return False
return not any(endpoint.startswith("/v1/realtime") for endpoint in model_info.get("supported_endpoints") or ())
endpoints: Final = (
() if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or ()
)
return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints)
_TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"}
@ -819,8 +820,9 @@ async def realtime_health_check(
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's
cost map lists the model under a mode realtime never serves: OpenAI checks the key first, and Live and
transcription models are served through other endpoints
cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live
and transcription models are served through other endpoints. The listed `mode` is not read, since a
deployment's own `mode: realtime` replaces it when the router registers the deployment
Raises:
Exception - if the connection is not successful or if OpenAI's first server event is any other error
"""

View file

@ -504,6 +504,18 @@ _OPENAI_SESSION_CREATED_EVENT: Final = (
)
def _register_as_realtime_deployment(model: str) -> None:
litellm.Router(
model_list=[
{
"model_name": "realtime-deployment",
"litellm_params": {"model": f"openai/{model}", "api_key": "sk-real"},
"model_info": {"mode": "realtime"},
}
]
)
class _CapturingConnect:
def __init__(self, connection: _ServerEvents | None = None) -> None:
self.url: str | None = None
@ -791,6 +803,7 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_would_fail_on(
local_model_cost_map, model: str, first_event: str, expected_exception: type[Exception], expected_message: str
):
_register_as_realtime_deployment(model)
connect: Final = _CapturingConnect(_ScriptedConnection(first_event))
with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised:
await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")