mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): read the cost map's listed endpoints, not its mode, since registering a realtime deployment overwrites the mode
This commit is contained in:
parent
42dff600f7
commit
69b2aa1339
2 changed files with 21 additions and 6 deletions
|
|
@ -746,14 +746,15 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
|
|||
return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
|
||||
|
||||
|
||||
_REALTIME_SERVED_MODES: Final = frozenset({"realtime", "audio_transcription"})
|
||||
_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions")
|
||||
|
||||
|
||||
def _cost_map_lists_outside_realtime(model: str) -> bool:
|
||||
model_info: Final = lookup_model_info_for_registration(model, "openai")
|
||||
if isinstance(model_info, ModelInfoLookupMiss) or model_info["mode"] in _REALTIME_SERVED_MODES:
|
||||
return False
|
||||
return not any(endpoint.startswith("/v1/realtime") for endpoint in model_info.get("supported_endpoints") or ())
|
||||
endpoints: Final = (
|
||||
() if isinstance(model_info, ModelInfoLookupMiss) else model_info.get("supported_endpoints") or ()
|
||||
)
|
||||
return bool(endpoints) and not any(endpoint.startswith(_REALTIME_SERVED_ENDPOINTS) for endpoint in endpoints)
|
||||
|
||||
|
||||
_TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"}
|
||||
|
|
@ -819,8 +820,9 @@ async def realtime_health_check(
|
|||
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
|
||||
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
|
||||
the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's
|
||||
cost map lists the model under a mode realtime never serves: OpenAI checks the key first, and Live and
|
||||
transcription models are served through other endpoints
|
||||
cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live
|
||||
and transcription models are served through other endpoints. The listed `mode` is not read, since a
|
||||
deployment's own `mode: realtime` replaces it when the router registers the deployment
|
||||
Raises:
|
||||
Exception - if the connection is not successful or if OpenAI's first server event is any other error
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -504,6 +504,18 @@ _OPENAI_SESSION_CREATED_EVENT: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _register_as_realtime_deployment(model: str) -> None:
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "realtime-deployment",
|
||||
"litellm_params": {"model": f"openai/{model}", "api_key": "sk-real"},
|
||||
"model_info": {"mode": "realtime"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class _CapturingConnect:
|
||||
def __init__(self, connection: _ServerEvents | None = None) -> None:
|
||||
self.url: str | None = None
|
||||
|
|
@ -791,6 +803,7 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
|
|||
async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_would_fail_on(
|
||||
local_model_cost_map, model: str, first_event: str, expected_exception: type[Exception], expected_message: str
|
||||
):
|
||||
_register_as_realtime_deployment(model)
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(first_event))
|
||||
with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised:
|
||||
await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue