diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 0d714a08b80..79032974878 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -8,7 +8,6 @@ import ssl from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, cast -from openai.types.realtime import RealtimeError, RealtimeErrorEvent from pydantic import TypeAdapter from litellm._logging import redact_string, verbose_logger @@ -31,6 +30,7 @@ from ..openai import OpenAIChatCompletion if TYPE_CHECKING: from fastapi import WebSocket + from openai.types.realtime import RealtimeError from websockets.asyncio.client import ClientConnection _SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object]) @@ -38,18 +38,22 @@ _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model" _CREDENTIALS_REJECTED: Final = "invalid_api_key" -def first_event_error(first_event: str | bytes) -> RealtimeError | None: +def first_event_error(first_event: str | bytes) -> "RealtimeError | None": event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event) if event.get("type") != "error": return None + from openai.types.realtime import RealtimeErrorEvent + return RealtimeErrorEvent.model_validate(event).error async def confirm_session_started( - connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool + connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool ) -> Literal[True]: error: Final = first_event_error(await connection.recv()) - if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known): + if error is None or ( + error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime + ): return True if error.code == _CREDENTIALS_REJECTED or not sent_credentials: raise AuthenticationError(message=error.message, llm_provider="openai", model=model) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 4b92371b4b8..4f12de61cd9 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -31,7 +31,12 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference +from litellm.utils import ( + ModelInfoLookupMiss, + ProviderConfigManager, + lookup_model_info_for_registration, + resolve_api_key_secret_reference, +) from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -741,6 +746,16 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str) return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ()) +_REALTIME_SERVED_MODES: Final = frozenset({"realtime", "audio_transcription"}) + + +def _cost_map_lists_outside_realtime(model: str) -> bool: + model_info: Final = lookup_model_info_for_registration(model, "openai") + if isinstance(model_info, ModelInfoLookupMiss) or model_info["mode"] in _REALTIME_SERVED_MODES: + return False + return not any(endpoint.startswith("/v1/realtime") for endpoint in model_info.get("supported_endpoints") or ()) + + _TRANSCRIPTION_QUERY_PARAMS: Final[RealtimeQueryParams] = {"intent": "transcription"} @@ -803,8 +818,9 @@ async def realtime_health_check( Returns: bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports - the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as - healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints + the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's + cost map lists the model under a mode realtime never serves: OpenAI checks the key first, and Live and + transcription models are served through other endpoints Raises: Exception - if the connection is not successful or if OpenAI's first server event is any other error """ @@ -905,7 +921,10 @@ async def realtime_health_check( if custom_llm_provider != "openai" or not is_openai_backed_api_base(url): return True return await confirm_session_started( - connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None + connection, + model, + sent_credentials=bool(auth_headers), + model_listed_outside_realtime=_cost_map_lists_outside_realtime(model), ) diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 90d485145a4..789f51778f1 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -490,9 +490,9 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = ( '{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", ' '"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' ) -_OPENAI_UNKNOWN_MODEL_EVENT: Final = ( +_OPENAI_CHAT_MODEL_ON_REALTIME_EVENT: Final = ( '{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", ' - '"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}' + '"message": "Model \\"gpt-5.5\\" is not supported in realtime mode.", "param": null, "event_id": null}}' ) _OPENAI_MODEL_NOT_FOUND_EVENT: Final = ( '{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", ' @@ -761,14 +761,13 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives(): @pytest.mark.asyncio +@pytest.mark.parametrize("model", ["gpt-live-1", "gpt-4o-transcribe", "gpt-audio", "gpt-live-unreleased"]) async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted( - local_model_cost_map, + local_model_cost_map, model: str ): connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT)) with patch("websockets.connect", connect): - assert await realtime_main.realtime_health_check( - model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real" - ) + assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real") @pytest.mark.asyncio @@ -776,10 +775,10 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim ("model", "first_event", "expected_exception", "expected_message"), [ ( - "realtime-2.1", - _OPENAI_UNKNOWN_MODEL_EVENT, + "gpt-5.5", + _OPENAI_CHAT_MODEL_ON_REALTIME_EVENT, litellm.BadRequestError, - 'Model "realtime-2.1" is not supported in realtime mode', + 'Model "gpt-5.5" is not supported in realtime mode', ), ( "gpt-realtime-2.1x", diff --git a/tests/unit/test_lazy_imports.py b/tests/unit/test_lazy_imports.py index 0416020a9b5..43aee4616d2 100644 --- a/tests/unit/test_lazy_imports.py +++ b/tests/unit/test_lazy_imports.py @@ -47,7 +47,7 @@ def test_import_litellm_does_not_load_fastapi_or_bpe_table(): [ sys.executable, "-c", - "import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding') if m in sys.modules))", + "import sys, litellm; print(','.join(m for m in ('fastapi','starlette','litellm.proxy.proxy_cli','litellm.litellm_core_utils.default_encoding','openai.types.realtime') if m in sys.modules))", ], check=True, capture_output=True,