mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge a7c2be7685 into 57377c9249
This commit is contained in:
commit
474d272e8e
4 changed files with 171 additions and 10 deletions
|
|
@ -313,6 +313,7 @@ REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = int(_max_size_env) if _max_si
|
|||
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float(
|
||||
os.getenv("REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", "20.0")
|
||||
)
|
||||
REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS: Final = 10.0
|
||||
|
||||
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
|
||||
|
|
|
|||
|
|
@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
|
|||
This requires websockets, and is currently only supported on LiteLLM Proxy.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import ssl
|
||||
from typing import Any, Final, cast
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
||||
|
||||
from openai.types.realtime import RealtimeError, RealtimeErrorEvent
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import _redact_string, verbose_logger
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.exceptions import AuthenticationError, BadRequestError, InternalServerError, Timeout
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
|
|
@ -20,6 +26,43 @@ from ....litellm_core_utils.realtime_streaming import (
|
|||
from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
|
||||
from ..openai import OpenAIChatCompletion
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def first_event_error(first_event: str | bytes) -> RealtimeError | None:
|
||||
event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event)
|
||||
if event.get("type") != "error":
|
||||
return None
|
||||
return RealtimeErrorEvent.model_validate(event).error
|
||||
|
||||
|
||||
def first_event_exception(error: RealtimeError, model: str) -> Exception:
|
||||
match error:
|
||||
case RealtimeError(code="invalid_api_key"):
|
||||
return AuthenticationError(message=error.message, llm_provider="openai", model=model)
|
||||
case RealtimeError(type="server_error"):
|
||||
return InternalServerError(message=error.message, llm_provider="openai", model=model)
|
||||
case _:
|
||||
return BadRequestError(message=error.message, model=model, llm_provider="openai")
|
||||
|
||||
|
||||
async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]:
|
||||
try:
|
||||
first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise Timeout(
|
||||
message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake",
|
||||
model=model,
|
||||
llm_provider="openai",
|
||||
) from None
|
||||
error: Final = first_event_error(first_event)
|
||||
if error is None:
|
||||
return True
|
||||
raise first_event_exception(error, model)
|
||||
|
||||
|
||||
class OpenAIRealtime(OpenAIChatCompletion):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import litellm
|
|||
from litellm.constants import (
|
||||
AZURE_OPENAI_AUDIO_PROVIDERS,
|
||||
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
|
||||
REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS,
|
||||
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
|
||||
request_timeout,
|
||||
)
|
||||
|
|
@ -37,7 +38,7 @@ from ..llms.azure.common_utils import get_azure_ad_token
|
|||
from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client
|
||||
from ..llms.bedrock.realtime.handler import BedrockRealtime
|
||||
from ..llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
|
||||
from ..llms.openai.realtime.handler import OpenAIRealtime
|
||||
from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started
|
||||
from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
|
||||
from ..llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from ..llms.xai.realtime.handler import XAIRealtime
|
||||
|
|
@ -624,6 +625,7 @@ async def _realtime_health_check(
|
|||
api_version: str | None = None,
|
||||
realtime_protocol: str | None = None,
|
||||
model_params: dict | None = None,
|
||||
first_event_timeout_seconds: float = REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS,
|
||||
):
|
||||
"""
|
||||
Health check for realtime API - tries connection to the realtime API websocket
|
||||
|
|
@ -639,9 +641,12 @@ async def _realtime_health_check(
|
|||
without the OpenAI-Beta header is bridged to, with transcription-only models adding intent=transcription
|
||||
|
||||
Returns:
|
||||
bool - True if connection is successful, False otherwise
|
||||
bool - True once the connection is open, and for OpenAI once the first server event is not an error,
|
||||
since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
|
||||
the failure in its first server event
|
||||
Raises:
|
||||
Exception - if the connection is not successful
|
||||
Exception - if the connection is not successful, if OpenAI's first server event is an error, or if
|
||||
OpenAI sends no server event within first_event_timeout_seconds
|
||||
"""
|
||||
import websockets
|
||||
|
||||
|
|
@ -721,5 +726,7 @@ async def _realtime_health_check(
|
|||
additional_headers=auth_headers,
|
||||
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
|
||||
ssl=ssl_context,
|
||||
):
|
||||
return True
|
||||
) as connection:
|
||||
if custom_llm_provider != "openai":
|
||||
return True
|
||||
return await confirm_session_started(connection, model, first_event_timeout_seconds)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ from typing import Final
|
|||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from websockets.exceptions import ConnectionClosedError
|
||||
from websockets.frames import Close
|
||||
|
||||
import litellm
|
||||
from litellm.models.credentials import CredentialItem
|
||||
|
|
@ -223,17 +225,18 @@ def test_client_secret_forwards_nested_transcription_model_untouched(monkeypatch
|
|||
|
||||
|
||||
class _CapturingConnect:
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, connection: object | None = None) -> None:
|
||||
self.url: str | None = None
|
||||
self.kwargs: dict[str, object] = {}
|
||||
self._connection: Final = connection if connection is not None else MagicMock()
|
||||
|
||||
def __call__(self, url: str, **kwargs: object) -> "_CapturingConnect":
|
||||
self.url = url
|
||||
self.kwargs = kwargs
|
||||
return self
|
||||
|
||||
async def __aenter__(self) -> MagicMock:
|
||||
return MagicMock()
|
||||
async def __aenter__(self) -> object:
|
||||
return self._connection
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
|
|
@ -298,7 +301,7 @@ async def test_bearer_health_check_sends_stored_credential_as_bearer_token(
|
|||
)
|
||||
],
|
||||
)
|
||||
connect = _CapturingConnect()
|
||||
connect = _CapturingConnect(_ScriptedConnection(_OPENAI_SESSION_CREATED_EVENT))
|
||||
with patch("websockets.connect", connect):
|
||||
assert await realtime_main._realtime_health_check(
|
||||
model=model,
|
||||
|
|
@ -412,6 +415,113 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch):
|
|||
assert connect.url == _AZURE_BETA_HEALTH_URL
|
||||
|
||||
|
||||
class _ScriptedConnection:
|
||||
def __init__(self, *frames: str) -> None:
|
||||
self._frames: Final = iter(frames)
|
||||
|
||||
async def recv(self) -> str:
|
||||
return next(self._frames)
|
||||
|
||||
|
||||
class _SilentConnection:
|
||||
async def recv(self) -> str:
|
||||
await asyncio.Event().wait()
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
class _ConnectionClosedBeforeAnyEvent:
|
||||
async def recv(self) -> str:
|
||||
raise ConnectionClosedError(Close(3000, "invalid_api_key"), None)
|
||||
|
||||
|
||||
_OPENAI_INVALID_API_KEY_EVENT: Final = (
|
||||
'{"type": "error", "event_id": "event_1", "error": {"type": "invalid_request_error", "code": "invalid_api_key", '
|
||||
'"message": "Incorrect API key provided: sk-proj-****0000. You can find your API key at '
|
||||
'https://platform.openai.com/account/api-keys.", "param": null, "event_id": null}}'
|
||||
)
|
||||
_OPENAI_MISSING_AUTH_EVENT: Final = (
|
||||
'{"type": "error", "event_id": "event_2", "error": {"type": "invalid_request_error", "code": null, '
|
||||
'"message": "Missing bearer or basic authentication in header", "param": null, "event_id": null}}'
|
||||
)
|
||||
_OPENAI_SERVER_ERROR_EVENT: Final = (
|
||||
'{"type": "error", "event_id": "event_3", '
|
||||
'"error": {"type": "server_error", "code": null, "message": "The server had an error", "param": null}}'
|
||||
)
|
||||
_OPENAI_SESSION_CREATED_EVENT: Final = (
|
||||
'{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("first_event", "expected_exception", "expected_status", "expected_message"),
|
||||
[
|
||||
(_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"),
|
||||
(_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"),
|
||||
(_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"),
|
||||
],
|
||||
)
|
||||
async def test_openai_health_check_reports_the_first_error_event_as_unhealthy(
|
||||
first_event: str, expected_exception: type[Exception], expected_status: int, expected_message: str
|
||||
):
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(first_event))
|
||||
with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised:
|
||||
await realtime_main._realtime_health_check(
|
||||
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-wrong"
|
||||
)
|
||||
assert raised.value.status_code == expected_status
|
||||
assert expected_message in str(raised.value)
|
||||
assert connect.url == "wss://api.openai.com/v1/realtime?model=gpt-realtime"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_is_unhealthy():
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_MISSING_AUTH_EVENT))
|
||||
with patch("websockets.connect", connect), pytest.raises(litellm.BadRequestError) as raised:
|
||||
await realtime_main._realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key=None)
|
||||
assert connect.kwargs["additional_headers"] == {}
|
||||
assert "Missing bearer or basic authentication" in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_is_healthy_once_session_created_arrives():
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_SESSION_CREATED_EVENT))
|
||||
with patch("websockets.connect", connect):
|
||||
assert await realtime_main._realtime_health_check(
|
||||
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time():
|
||||
connect: Final = _CapturingConnect(_SilentConnection())
|
||||
with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised:
|
||||
await realtime_main._realtime_health_check(
|
||||
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01
|
||||
)
|
||||
assert raised.value.status_code == 408
|
||||
assert "no server event within 0.01 seconds" in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_is_unhealthy_when_the_socket_closes_before_any_event():
|
||||
connect: Final = _CapturingConnect(_ConnectionClosedBeforeAnyEvent())
|
||||
with patch("websockets.connect", connect), pytest.raises(ConnectionClosedError):
|
||||
await realtime_main._realtime_health_check(
|
||||
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-wrong"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_xai_health_check_trusts_the_handshake_without_reading_a_first_event():
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection())
|
||||
with patch("websockets.connect", connect):
|
||||
assert await realtime_main._realtime_health_check(
|
||||
model="grok-voice-latest", custom_llm_provider="xai", api_key="sk-wrong"
|
||||
)
|
||||
assert connect.url == "wss://api.x.ai/v1/realtime?model=grok-voice-latest"
|
||||
|
||||
|
||||
class _ConnectThatStopsAfterCapturingTheUrl:
|
||||
url: str | None = None
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue