mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): read a realtime gateway's first event and keep Live deployments listing /v1/live healthy
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Behind a custom api_base the OpenAI realtime health check now waits up to 3 seconds for the first event, so a gateway that passes on OpenAI's invalid_api_key error reports unhealthy while a silent gateway stays healthy. /v1/live counts as a realtime endpoint for the invalid_model rule.
This commit is contained in:
parent
69b2aa1339
commit
393d9c63ca
3 changed files with 89 additions and 11 deletions
|
|
@ -4,6 +4,7 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
|
|||
This requires websockets, and is currently only supported on LiteLLM Proxy.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import ssl
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
|
|
@ -47,10 +48,10 @@ def first_event_error(first_event: str | bytes) -> "RealtimeError | None":
|
|||
return RealtimeErrorEvent.model_validate(event).error
|
||||
|
||||
|
||||
async def confirm_session_started(
|
||||
connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
|
||||
def _confirm_first_event(
|
||||
first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool
|
||||
) -> Literal[True]:
|
||||
error: Final = first_event_error(await connection.recv())
|
||||
error: Final = first_event_error(first_event)
|
||||
if error is None or (
|
||||
error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime
|
||||
):
|
||||
|
|
@ -60,6 +61,26 @@ async def confirm_session_started(
|
|||
raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai")
|
||||
|
||||
|
||||
async def confirm_session_started(
|
||||
connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
|
||||
) -> Literal[True]:
|
||||
return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime)
|
||||
|
||||
|
||||
async def confirm_gateway_session_started(
|
||||
connection: "ClientConnection",
|
||||
model: str,
|
||||
sent_credentials: bool,
|
||||
model_listed_outside_realtime: bool,
|
||||
first_event_wait_seconds: float,
|
||||
) -> Literal[True]:
|
||||
try:
|
||||
first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
return True
|
||||
return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime)
|
||||
|
||||
|
||||
class OpenAIRealtime(OpenAIChatCompletion):
|
||||
"""
|
||||
Base handler for OpenAI-compatible realtime WebSocket connections.
|
||||
|
|
|
|||
|
|
@ -49,7 +49,11 @@ from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_co
|
|||
from ..llms.custom_httpx.http_handler import realtime_ssl_for_url
|
||||
from ..llms.openai.common_utils import is_openai_backed_api_base
|
||||
from ..llms.openai.live.handler import OpenAILiveSessions
|
||||
from ..llms.openai.realtime.handler import OpenAIRealtime, confirm_session_started
|
||||
from ..llms.openai.realtime.handler import (
|
||||
OpenAIRealtime,
|
||||
confirm_gateway_session_started,
|
||||
confirm_session_started,
|
||||
)
|
||||
from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async
|
||||
from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model
|
||||
from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config
|
||||
|
|
@ -746,7 +750,8 @@ def _is_transcription_only_realtime_model(model: str, custom_llm_provider: str)
|
|||
return "/v1/realtime/transcription_sessions" in (model_info.get("supported_endpoints") or ())
|
||||
|
||||
|
||||
_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions")
|
||||
_REALTIME_SERVED_ENDPOINTS: Final = ("/v1/realtime", "/v1/audio/transcriptions", "/v1/live")
|
||||
_GATEWAY_FIRST_EVENT_WAIT_SECONDS: Final = 3.0
|
||||
|
||||
|
||||
def _cost_map_lists_outside_realtime(model: str) -> bool:
|
||||
|
|
@ -802,6 +807,7 @@ async def realtime_health_check(
|
|||
api_version: str | None = None,
|
||||
realtime_protocol: str | None = None,
|
||||
model_params: Mapping[str, object] | None = None,
|
||||
gateway_first_event_wait_seconds: float = _GATEWAY_FIRST_EVENT_WAIT_SECONDS,
|
||||
) -> bool:
|
||||
"""
|
||||
Health check for realtime API - tries connection to the realtime API websocket
|
||||
|
|
@ -819,7 +825,9 @@ async def realtime_health_check(
|
|||
Returns:
|
||||
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
|
||||
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
|
||||
the failure in its first server event. An `invalid_model` error also counts as healthy unless litellm's
|
||||
the failure in its first server event. Behind any other OpenAI api_base the first event is read for
|
||||
`gateway_first_event_wait_seconds` only, and a gateway that stays silent that long counts as healthy.
|
||||
An `invalid_model` error also counts as healthy unless litellm's
|
||||
cost map lists only endpoints realtime never serves for the model: OpenAI checks the key first, and Live
|
||||
and transcription models are served through other endpoints. The listed `mode` is not read, since a
|
||||
deployment's own `mode: realtime` replaces it when the router registers the deployment
|
||||
|
|
@ -920,8 +928,16 @@ async def realtime_health_check(
|
|||
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
|
||||
ssl=realtime_ssl_for_url(url),
|
||||
) as connection:
|
||||
if custom_llm_provider != "openai" or not is_openai_backed_api_base(url):
|
||||
if custom_llm_provider != "openai":
|
||||
return True
|
||||
if not is_openai_backed_api_base(url):
|
||||
return await confirm_gateway_session_started(
|
||||
connection,
|
||||
model,
|
||||
sent_credentials=bool(auth_headers),
|
||||
model_listed_outside_realtime=_cost_map_lists_outside_realtime(model),
|
||||
first_event_wait_seconds=gateway_first_event_wait_seconds,
|
||||
)
|
||||
return await confirm_session_started(
|
||||
connection,
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -504,13 +504,16 @@ _OPENAI_SESSION_CREATED_EVENT: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _register_as_realtime_deployment(model: str) -> None:
|
||||
def _register_as_realtime_deployment(model: str, *supported_endpoints: str) -> None:
|
||||
litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "realtime-deployment",
|
||||
"litellm_params": {"model": f"openai/{model}", "api_key": "sk-real"},
|
||||
"model_info": {"mode": "realtime"},
|
||||
"model_info": {
|
||||
"mode": "realtime",
|
||||
**({"supported_endpoints": list(supported_endpoints)} if supported_endpoints else {}),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
|
@ -782,6 +785,18 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
|
|||
assert await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_is_healthy_for_a_live_deployment_that_lists_the_live_sessions_endpoint(
|
||||
local_model_cost_map,
|
||||
):
|
||||
_register_as_realtime_deployment("gpt-live-1", "/v1/live/sessions")
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT))
|
||||
with patch("websockets.connect", connect):
|
||||
assert await realtime_main.realtime_health_check(
|
||||
model="gpt-live-1", custom_llm_provider="openai", api_key="sk-real"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "first_event", "expected_exception", "expected_message"),
|
||||
|
|
@ -815,7 +830,7 @@ async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_w
|
|||
("deployment_api_base", "global_api_base"),
|
||||
[("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")],
|
||||
)
|
||||
async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake(
|
||||
async def test_openai_health_check_through_a_compatible_gateway_is_healthy_when_it_sends_no_first_event(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None
|
||||
):
|
||||
monkeypatch.setattr(litellm, "api_base", global_api_base)
|
||||
|
|
@ -823,13 +838,39 @@ async def test_openai_health_check_through_a_compatible_gateway_trusts_the_hands
|
|||
with patch("websockets.connect", connect):
|
||||
assert await asyncio.wait_for(
|
||||
realtime_main.realtime_health_check(
|
||||
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base
|
||||
model="gpt-realtime",
|
||||
custom_llm_provider="openai",
|
||||
api_key="sk-real",
|
||||
api_base=deployment_api_base,
|
||||
gateway_first_event_wait_seconds=0.01,
|
||||
),
|
||||
timeout=5,
|
||||
)
|
||||
assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("connection", "expected_exception"),
|
||||
[
|
||||
(_ScriptedConnection(_OPENAI_INVALID_API_KEY_EVENT), litellm.AuthenticationError),
|
||||
(_ConnectionClosedBeforeAnyEvent(), ConnectionClosedError),
|
||||
],
|
||||
)
|
||||
async def test_openai_health_check_through_a_compatible_gateway_is_unhealthy_when_it_rejects_the_session(
|
||||
connection: _ServerEvents, expected_exception: type[Exception]
|
||||
):
|
||||
connect: Final = _CapturingConnect(connection)
|
||||
with patch("websockets.connect", connect), pytest.raises(expected_exception):
|
||||
await realtime_main.realtime_health_check(
|
||||
model="gpt-realtime",
|
||||
custom_llm_provider="openai",
|
||||
api_key="sk-wrong",
|
||||
api_base="https://realtime-gateway.example/v1",
|
||||
)
|
||||
assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives():
|
||||
connect: Final = _CapturingConnect(_SilentConnection())
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue