mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(health): keep invalid_model healthy unless the cost map lists the model outside realtime, and load realtime types lazily
This commit is contained in:
parent
d8eaff8a0f
commit
42dff600f7
4 changed files with 40 additions and 18 deletions
|
|
@ -8,7 +8,6 @@ import ssl
|
|||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
|
||||
from openai.types.realtime import RealtimeError, RealtimeErrorEvent
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import redact_string, verbose_logger
|
||||
|
|
@ -31,6 +30,7 @@ from ..openai import OpenAIChatCompletion
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import WebSocket
|
||||
from openai.types.realtime import RealtimeError
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
|
@ -38,18 +38,22 @@ _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model"
|
|||
_CREDENTIALS_REJECTED: Final = "invalid_api_key"
|
||||
|
||||
|
||||
def first_event_error(first_event: str | bytes) -> RealtimeError | None:
|
||||
def first_event_error(first_event: str | bytes) -> "RealtimeError | None":
|
||||
event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event)
|
||||
if event.get("type") != "error":
|
||||
return None
|
||||
from openai.types.realtime import RealtimeErrorEvent
|
||||
|
||||
return RealtimeErrorEvent.model_validate(event).error
|
||||
|
||||
|
||||
async def confirm_session_started(
|
||||
connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool
|
||||
connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
|
||||
) -> Literal[True]:
|
||||
error: Final = first_event_error(await connection.recv())
|
||||
if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known):
|
||||
if error is None or (
|
||||
error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime
|
||||
):
|
||||
return True
|
||||
if error.code == _CREDENTIALS_REJECTED or not sent_credentials:
|
||||
raise AuthenticationError(message=error.message, llm_provider="openai", model=model)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,12 @@ from litellm.types.realtime import (
|
|||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import CallTypes, LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference
|
||||
from litellm.utils import (
|
||||
ModelInfoLookupMiss,
|
||||
ProviderConfigManager,
|
||||
lookup_model_info_for_registration,
|
||||
resolve_api_key_secret_reference,
|
||||
)
|
||||
|
||||
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
|
||||
from ..litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
|
|
@ -741,6 +746,16 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
|
|||
return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
|
||||
|
||||
|
||||
_REALTIME_SERVED_MODES: Final = frozenset({"realtime", "audio_transcription"})
|
||||
|
||||
|
||||
def _cost_map_lists_outside_realtime(model: str) -> bool:
|
||||
model_info: Final = lookup_model_info_for_registration(model, "openai")
|
||||
if isinstance(model_info, ModelInfoLookupMiss) or model_info["mode"] in _REALTIME_SERVED_MODES:
|
||||
return False
|
||||
return not any(endpoint.startswith("/v1/realtime") for endpoint in model_info.get("supported_endpoints") or ())
|
||||
|
||||
|
||||
_TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"}
|
||||
|
||||
|
||||
|
|
@ -803,8 +818,9 @@ async def realtime_health_check(
|
|||
Returns:
|
||||
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
|
||||
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
|
||||
the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as
|
||||
healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints
|
||||
the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's
|
||||
cost map lists the model under a mode realtime never serves: OpenAI checks the key first, and Live and
|
||||
transcription models are served through other endpoints
|
||||
Raises:
|
||||
Exception - if the connection is not successful or if OpenAI's first server event is any other error
|
||||
"""
|
||||
|
|
@ -905,7 +921,10 @@ async def realtime_health_check(
|
|||
if custom_llm_provider != "openai" or not is_openai_backed_api_base(url):
|
||||
return True
|
||||
return await confirm_session_started(
|
||||
connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None
|
||||
connection,
|
||||
model,
|
||||
sent_credentials=bool(auth_headers),
|
||||
model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -490,9 +490,9 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = (
|
|||
'{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", '
|
||||
'"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
|
||||
)
|
||||
_OPENAI_UNKNOWN_MODEL_EVENT: Final = (
|
||||
_OPENAI_CHAT_MODEL_ON_REALTIME_EVENT: Final = (
|
||||
'{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", '
|
||||
'"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
|
||||
'"message": "Model \\"gpt-5.5\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
|
||||
)
|
||||
_OPENAI_MODEL_NOT_FOUND_EVENT: Final = (
|
||||
'{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", '
|
||||
|
|
@ -761,14 +761,13 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("model", ["gpt-live-1", "gpt-4o-transcribe", "gpt-audio", "gpt-live-unreleased"])
|
||||
async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted(
|
||||
local_model_cost_map,
|
||||
local_model_cost_map, model: str
|
||||
):
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT))
|
||||
with patch("websockets.connect", connect):
|
||||
assert await realtime_main.realtime_health_check(
|
||||
model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real"
|
||||
)
|
||||
assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -776,10 +775,10 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
|
|||
("model", "first_event", "expected_exception", "expected_message"),
|
||||
[
|
||||
(
|
||||
"realtime-2.1",
|
||||
_OPENAI_UNKNOWN_MODEL_EVENT,
|
||||
"gpt-5.5",
|
||||
_OPENAI_CHAT_MODEL_ON_REALTIME_EVENT,
|
||||
litellm.BadRequestError,
|
||||
'Model "realtime-2.1" is not supported in realtime mode',
|
||||
'Model "gpt-5.5" is not supported in realtime mode',
|
||||
),
|
||||
(
|
||||
"gpt-realtime-2.1x",
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ def test_import_litellm_does_not_load_fastapi_or_bpe_table():
|
|||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding') if m in sys.modules))",
|
||||
"import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding','openai.types.realtime') if m in sys.modules))",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue