mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): respect the global api_base, report a missing key as 401, keep invalid_model healthy only for known models
Drop the separate first-event timeout so model_info.health_check_timeout governs the wait, and keep the shared stream error table unchanged by raising AuthenticationError for invalid_api_key in the realtime check
This commit is contained in:
parent
678104a8e5
commit
d8eaff8a0f
5 changed files with 79 additions and 45 deletions
|
|
@ -416,7 +416,6 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
|
|||
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
|
||||
)
|
||||
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
|
||||
REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS: Final = 10.0
|
||||
|
||||
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
|
|||
This requires websockets, and is currently only supported on LiteLLM Proxy.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import ssl
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast
|
||||
|
|
@ -14,7 +13,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
from litellm._logging import redact_string, verbose_logger
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.exceptions import Timeout
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.responses.streaming_iterator import map_stream_error_to_exception
|
||||
from litellm.types.realtime import RealtimeQueryParams
|
||||
|
||||
|
|
@ -36,6 +35,7 @@ if TYPE_CHECKING:
|
|||
|
||||
_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
|
||||
_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model"
|
||||
_CREDENTIALS_REJECTED: Final = "invalid_api_key"
|
||||
|
||||
|
||||
def first_event_error(first_event: str | bytes) -> RealtimeError | None:
|
||||
|
|
@ -45,18 +45,14 @@ def first_event_error(first_event: str | bytes) -> RealtimeError | None:
|
|||
return RealtimeErrorEvent.model_validate(event).error
|
||||
|
||||
|
||||
async def confirm_session_started(connection: "ClientConnection", model: str, timeout_seconds: float) -> Literal[True]:
|
||||
try:
|
||||
first_event: Final = await asyncio.wait_for(connection.recv(), timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise Timeout(
|
||||
message=f"OpenAI realtime sent no server event within {timeout_seconds} seconds of the handshake",
|
||||
model=model,
|
||||
llm_provider="openai",
|
||||
) from None
|
||||
error: Final = first_event_error(first_event)
|
||||
if error is None or error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL:
|
||||
async def confirm_session_started(
|
||||
connection: "ClientConnection", model: str, sent_credentials: bool, model_is_known: bool
|
||||
) -> Literal[True]:
|
||||
error: Final = first_event_error(await connection.recv())
|
||||
if error is None or (error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and model_is_known):
|
||||
return True
|
||||
if error.code == _CREDENTIALS_REJECTED or not sent_credentials:
|
||||
raise AuthenticationError(message=error.message, llm_provider="openai", model=model)
|
||||
raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import litellm
|
|||
from litellm.constants import (
|
||||
AZURE_OPENAI_AUDIO_PROVIDERS,
|
||||
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
|
||||
REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS,
|
||||
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
|
||||
request_timeout,
|
||||
)
|
||||
|
|
@ -32,7 +31,7 @@ from litellm.types.realtime import (
|
|||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import CallTypes, LlmProviders
|
||||
from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference
|
||||
from litellm.utils import ProviderConfigManager, get_model_cost_key, resolve_api_key_secret_reference
|
||||
|
||||
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
|
||||
from ..litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
|
|
@ -787,7 +786,6 @@ async def realtime_health_check(
|
|||
api_version: str | None = None,
|
||||
realtime_protocol: str | None = None,
|
||||
model_params: Mapping[str, object] | None = None,
|
||||
first_event_timeout_seconds: float = REALTIME_HEALTH_CHECK_FIRST_EVENT_TIMEOUT_SECONDS,
|
||||
) -> bool:
|
||||
"""
|
||||
Health check for realtime API - tries connection to the realtime API websocket
|
||||
|
|
@ -805,11 +803,10 @@ async def realtime_health_check(
|
|||
Returns:
|
||||
bool - True once the connection is open, and on OpenAI's own hosts once the first server event is not an
|
||||
error, since OpenAI accepts the websocket handshake with missing or invalid credentials and only reports
|
||||
the failure in its first server event. An `invalid_model` error also counts as healthy: OpenAI checks
|
||||
the key first, and Live and transcription models are served through other endpoints
|
||||
the failure in its first server event. An `invalid_model` error for a model litellm knows also counts as
|
||||
healthy: OpenAI checks the key first, and Live and transcription models are served through other endpoints
|
||||
Raises:
|
||||
Exception - if the connection is not successful, if OpenAI's first server event is any other error, or if
|
||||
OpenAI sends no server event within first_event_timeout_seconds
|
||||
Exception - if the connection is not successful or if OpenAI's first server event is any other error
|
||||
"""
|
||||
import websockets
|
||||
|
||||
|
|
@ -829,10 +826,11 @@ async def realtime_health_check(
|
|||
str | None, api_version or resolved_params.get("api_version")
|
||||
)
|
||||
url: str | None = None
|
||||
openai_api_base: Final = resolved_api_base or litellm.api_base or "https://api.openai.com/"
|
||||
health_check_api_key: Final = (
|
||||
await resolve_openai_bearer_token_async(
|
||||
static_api_key=openai_static_api_key(resolved_api_key),
|
||||
api_base=resolved_api_base,
|
||||
api_base=openai_api_base,
|
||||
litellm_params=resolved_params,
|
||||
)
|
||||
if custom_llm_provider == LlmProviders.OPENAI.value
|
||||
|
|
@ -857,10 +855,7 @@ async def realtime_health_check(
|
|||
query_params=azure_query_params,
|
||||
)
|
||||
elif custom_llm_provider == "openai":
|
||||
url = openai_realtime.construct_url(
|
||||
api_base=resolved_api_base or "https://api.openai.com/",
|
||||
query_params={"model": model},
|
||||
)
|
||||
url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model})
|
||||
elif custom_llm_provider == "xai":
|
||||
url = xai_realtime.construct_url(
|
||||
api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model}
|
||||
|
|
@ -909,7 +904,9 @@ async def realtime_health_check(
|
|||
) as connection:
|
||||
if custom_llm_provider != "openai" or not is_openai_backed_api_base(url):
|
||||
return True
|
||||
return await confirm_session_started(connection, model, first_event_timeout_seconds)
|
||||
return await confirm_session_started(
|
||||
connection, model, sent_credentials=bool(auth_headers), model_is_known=get_model_cost_key(model) is not None
|
||||
)
|
||||
|
||||
|
||||
_realtime_health_check = realtime_health_check
|
||||
|
|
|
|||
|
|
@ -228,7 +228,6 @@ async def drain_logging_tasks(timeout: float = 5.0) -> None:
|
|||
_ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType(
|
||||
{
|
||||
"server_error": 500,
|
||||
"invalid_api_key": 401,
|
||||
"rate_limit_exceeded": 429,
|
||||
"insufficient_quota": 429,
|
||||
"vector_store_timeout": 504,
|
||||
|
|
|
|||
|
|
@ -490,6 +490,15 @@ _OPENAI_LIVE_MODEL_ON_REALTIME_EVENT: Final = (
|
|||
'{"type": "error", "event_id": "event_7", "error": {"type": "invalid_request_error", "code": "invalid_model", '
|
||||
'"message": "Model \\"gpt-live-1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
|
||||
)
|
||||
_OPENAI_UNKNOWN_MODEL_EVENT: Final = (
|
||||
'{"type": "error", "event_id": "event_8", "error": {"type": "invalid_request_error", "code": "invalid_model", '
|
||||
'"message": "Model \\"realtime-2.1\\" is not supported in realtime mode.", "param": null, "event_id": null}}'
|
||||
)
|
||||
_OPENAI_MODEL_NOT_FOUND_EVENT: Final = (
|
||||
'{"type": "error", "event_id": "event_9", "error": {"type": "invalid_request_error", "code": "model_not_found", '
|
||||
'"message": "The model gpt-realtime-2.1x does not exist or you do not have access to it.", "param": null, '
|
||||
'"event_id": null}}'
|
||||
)
|
||||
_OPENAI_SESSION_CREATED_EVENT: Final = (
|
||||
'{"type": "session.created", "event_id": "event_4", "session": {"type": "realtime", "model": "gpt-realtime"}}'
|
||||
)
|
||||
|
|
@ -714,7 +723,6 @@ async def test_azure_health_check_honors_env_realtime_protocol(monkeypatch):
|
|||
("first_event", "expected_exception", "expected_status", "expected_message"),
|
||||
[
|
||||
(_OPENAI_INVALID_API_KEY_EVENT, litellm.AuthenticationError, 401, "Incorrect API key provided"),
|
||||
(_OPENAI_MISSING_AUTH_EVENT, litellm.BadRequestError, 400, "Missing bearer or basic authentication"),
|
||||
(_OPENAI_SERVER_ERROR_EVENT, litellm.InternalServerError, 500, "The server had an error"),
|
||||
(_OPENAI_RATE_LIMIT_EVENT, litellm.RateLimitError, 429, "Rate limit reached"),
|
||||
(_OPENAI_INSUFFICIENT_QUOTA_EVENT, litellm.RateLimitError, 429, "You exceeded your current quota"),
|
||||
|
|
@ -734,11 +742,12 @@ async def test_openai_health_check_reports_the_first_error_event_as_unhealthy(
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_is_unhealthy():
|
||||
async def test_openai_health_check_without_an_api_key_sends_no_auth_header_and_fails_authentication():
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_MISSING_AUTH_EVENT))
|
||||
with patch("websockets.connect", connect), pytest.raises(litellm.BadRequestError) as raised:
|
||||
with patch("websockets.connect", connect), pytest.raises(litellm.AuthenticationError) as raised:
|
||||
await realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key=None)
|
||||
assert connect.kwargs["additional_headers"] == {}
|
||||
assert raised.value.status_code == 401
|
||||
assert "Missing bearer or basic authentication" in str(raised.value)
|
||||
|
||||
|
||||
|
|
@ -752,7 +761,9 @@ async def test_openai_health_check_is_healthy_once_session_created_arrives():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted():
|
||||
async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtime_once_the_key_is_accepted(
|
||||
local_model_cost_map,
|
||||
):
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(_OPENAI_LIVE_MODEL_ON_REALTIME_EVENT))
|
||||
with patch("websockets.connect", connect):
|
||||
assert await realtime_main.realtime_health_check(
|
||||
|
|
@ -761,28 +772,60 @@ async def test_openai_health_check_is_healthy_for_a_model_served_outside_realtim
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake():
|
||||
@pytest.mark.parametrize(
|
||||
("model", "first_event", "expected_exception", "expected_message"),
|
||||
[
|
||||
(
|
||||
"realtime-2.1",
|
||||
_OPENAI_UNKNOWN_MODEL_EVENT,
|
||||
litellm.BadRequestError,
|
||||
'Model "realtime-2.1" is not supported in realtime mode',
|
||||
),
|
||||
(
|
||||
"gpt-realtime-2.1x",
|
||||
_OPENAI_MODEL_NOT_FOUND_EVENT,
|
||||
litellm.NotFoundError,
|
||||
"The model gpt-realtime-2.1x does not exist",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_openai_health_check_is_unhealthy_for_a_model_name_every_session_would_fail_on(
|
||||
local_model_cost_map, model: str, first_event: str, expected_exception: type[Exception], expected_message: str
|
||||
):
|
||||
connect: Final = _CapturingConnect(_ScriptedConnection(first_event))
|
||||
with patch("websockets.connect", connect), pytest.raises(expected_exception) as raised:
|
||||
await realtime_main.realtime_health_check(model=model, custom_llm_provider="openai", api_key="sk-real")
|
||||
assert expected_message in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_api_base", "global_api_base"),
|
||||
[("https://realtime-gateway.example/v1", None), (None, "https://realtime-gateway.example/v1")],
|
||||
)
|
||||
async def test_openai_health_check_through_a_compatible_gateway_trusts_the_handshake(
|
||||
monkeypatch: pytest.MonkeyPatch, deployment_api_base: str | None, global_api_base: str | None
|
||||
):
|
||||
monkeypatch.setattr(litellm, "api_base", global_api_base)
|
||||
connect: Final = _CapturingConnect(_SilentConnection())
|
||||
with patch("websockets.connect", connect):
|
||||
assert await realtime_main.realtime_health_check(
|
||||
model="gpt-realtime",
|
||||
custom_llm_provider="openai",
|
||||
api_key="sk-real",
|
||||
api_base="https://realtime-gateway.example/v1",
|
||||
first_event_timeout_seconds=0.01,
|
||||
assert await asyncio.wait_for(
|
||||
realtime_main.realtime_health_check(
|
||||
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", api_base=deployment_api_base
|
||||
),
|
||||
timeout=5,
|
||||
)
|
||||
assert connect.url == "wss://realtime-gateway.example/v1/realtime?model=gpt-realtime"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_health_check_is_unhealthy_when_no_first_event_arrives_in_time():
|
||||
async def test_openai_health_check_never_reports_healthy_before_the_first_event_arrives():
|
||||
connect: Final = _CapturingConnect(_SilentConnection())
|
||||
with patch("websockets.connect", connect), pytest.raises(litellm.Timeout) as raised:
|
||||
await realtime_main.realtime_health_check(
|
||||
model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real", first_event_timeout_seconds=0.01
|
||||
with patch("websockets.connect", connect), pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(
|
||||
realtime_main.realtime_health_check(model="gpt-realtime", custom_llm_provider="openai", api_key="sk-real"),
|
||||
timeout=0.05,
|
||||
)
|
||||
assert raised.value.status_code == 408
|
||||
assert "no server event within 0.01 seconds" in str(raised.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue