mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
Merge pull request #39841 from BerriAI/litellm_gate_openai_ws_passthrough
fix(proxy): gate the OpenAI websocket passthrough behind an explicit opt-in
This commit is contained in:
Mateo Wang
GitHub
commit
c52b53706e
9 changed files with 810 additions and 553 deletions
|
|
@ -2638,6 +2638,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
default=None,
|
||||
description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through",
|
||||
)
|
||||
enable_openai_websocket_passthrough: bool | None = Field(
|
||||
default=None,
|
||||
description="Serve the OpenAI pass-through WebSocket route, which relays frames to OpenAI under the proxy's own provider credential without reading them. Off by default.",
|
||||
)
|
||||
user_header_name: str | None = Field(
|
||||
None,
|
||||
description="[DEPRECATED] Use 'user_header_mappings' instead. When set, the header value is treated as the end user id unless overridden by user_header_mappings.",
|
||||
|
|
|
|||
|
|
@ -2352,6 +2352,13 @@ async def _backfill_null_user_email(
|
|||
return updated_row
|
||||
|
||||
|
||||
class UserNotFoundError(ValueError):
|
||||
"""The user row is provably absent, as opposed to merely unreadable, so a caller that reads a missing row as no user-level limits can key on it without also swallowing a database that would not answer."""
|
||||
|
||||
def __init__(self, user_id: str) -> None:
|
||||
super().__init__(f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call.")
|
||||
|
||||
|
||||
@log_db_metrics
|
||||
async def get_user_object(
|
||||
user_id: str | None,
|
||||
|
|
@ -2457,7 +2464,7 @@ async def get_user_object(
|
|||
value=None,
|
||||
last_db_access_time=last_db_access_time,
|
||||
)
|
||||
raise Exception
|
||||
raise UserNotFoundError(user_id=user_id)
|
||||
|
||||
if response.organization_memberships is not None and len(response.organization_memberships) > 0:
|
||||
# dump each organization membership to type LiteLLM_OrganizationMembershipTable
|
||||
|
|
@ -2493,7 +2500,9 @@ async def get_user_object(
|
|||
)
|
||||
|
||||
return _response
|
||||
except Exception as e: # if user not in db
|
||||
except UserNotFoundError:
|
||||
raise
|
||||
except Exception as e:
|
||||
_log_budget_lookup_failure("user", e)
|
||||
raise ValueError(
|
||||
f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call. Got error - {e}"
|
||||
|
|
@ -4155,6 +4164,79 @@ async def _granted_model_lists(
|
|||
)
|
||||
|
||||
|
||||
async def _user_object_or_none(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
prisma_client: PrismaClient,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
) -> LiteLLM_UserTable | None:
|
||||
try:
|
||||
return await get_user_object(
|
||||
user_id=valid_token.user_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
user_id_upsert=False,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
except UserNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
async def enforced_model_allowlists(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
) -> tuple[Sequence[str], ...]:
|
||||
"""One model allowlist per level that ``common_checks`` enforces on a request from this identity."""
|
||||
key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token)
|
||||
if prisma_client is None:
|
||||
return (key_models, tuple(valid_token.team_models or ()))
|
||||
team_object: Final = (
|
||||
None
|
||||
if valid_token.team_id is None
|
||||
else await get_team_object(
|
||||
team_id=valid_token.team_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
)
|
||||
user_object: Final = (
|
||||
None
|
||||
if team_object is not None
|
||||
else await _user_object_or_none(
|
||||
valid_token=valid_token,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
)
|
||||
project_object: Final = (
|
||||
None
|
||||
if valid_token.project_id is None
|
||||
else await get_project_object(
|
||||
project_id=valid_token.project_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
)
|
||||
return (
|
||||
key_models,
|
||||
team_object.models if team_object is not None else (),
|
||||
await _team_member_granted_models(
|
||||
valid_token=valid_token,
|
||||
team_object=team_object,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
),
|
||||
user_object.models if user_object is not None else (),
|
||||
project_object.models if project_object is not None else (),
|
||||
)
|
||||
|
||||
|
||||
async def collect_matched_model_access_groups(
|
||||
model: str | Sequence[str] | None,
|
||||
valid_token: UserAPIKeyAuth | None,
|
||||
|
|
|
|||
|
|
@ -13,14 +13,16 @@ import inspect
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import AsyncGenerator, Callable, Mapping
|
||||
from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, cast
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket
|
||||
from fastapi.responses import StreamingResponse
|
||||
from starlette.websockets import WebSocketState
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm import get_llm_provider
|
||||
|
|
@ -35,6 +37,7 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
|||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.passthrough.main import AsyncPassthroughStreamingResponse
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.auth_checks import enforced_model_allowlists
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.auth.user_api_key_auth import (
|
||||
|
|
@ -1773,7 +1776,7 @@ def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str]
|
|||
|
||||
|
||||
def get_vertex_pass_through_handler(
|
||||
call_type: Literal["discovery", "aiplatform"], # noqa: UP037 # ruff reports quoted Literal values here
|
||||
call_type: Literal["discovery", "aiplatform"],
|
||||
) -> BaseVertexAIPassThroughHandler:
|
||||
if call_type == "discovery":
|
||||
return VertexAIDiscoveryPassThroughHandler()
|
||||
|
|
@ -2340,9 +2343,102 @@ _OPENAI_WS_ALL_MODEL_ACCESS: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def _key_has_model_restrictions(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
scoped_models: Final = (*user_api_key_dict.models, *user_api_key_dict.team_models)
|
||||
return any(str(model) not in _OPENAI_WS_ALL_MODEL_ACCESS for model in scoped_models)
|
||||
def _has_model_restrictions(model_allowlists: tuple[Sequence[str], ...]) -> bool:
|
||||
return any(str(model) not in _OPENAI_WS_ALL_MODEL_ACCESS for allowlist in model_allowlists for model in allowlist)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OpenAIWebsocketRefusal:
|
||||
close_reason: str
|
||||
message: str
|
||||
|
||||
|
||||
class _OpenAIWebsocketErrorDetail(TypedDict):
|
||||
type: ReadOnly[Literal["invalid_request_error"]]
|
||||
message: ReadOnly[str]
|
||||
|
||||
|
||||
class _OpenAIWebsocketErrorFrame(TypedDict):
|
||||
type: ReadOnly[Literal["error"]]
|
||||
error: ReadOnly[_OpenAIWebsocketErrorDetail]
|
||||
|
||||
|
||||
_OPENAI_WS_DISABLED_REFUSAL: Final = _OpenAIWebsocketRefusal(
|
||||
close_reason="OpenAI websocket passthrough is disabled",
|
||||
message=(
|
||||
"OpenAI websocket passthrough is disabled on this gateway. A proxy admin can turn it on by "
|
||||
"setting general_settings.enable_openai_websocket_passthrough to true."
|
||||
),
|
||||
)
|
||||
|
||||
_OPENAI_WS_MODEL_RESTRICTED_REFUSAL: Final = _OpenAIWebsocketRefusal(
|
||||
close_reason="Keys with model restrictions cannot use OpenAI websocket passthrough",
|
||||
message=(
|
||||
"Keys with model restrictions cannot use OpenAI websocket passthrough, because this route "
|
||||
"relays frames to the provider without reading which model they ask for."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _is_openai_websocket_passthrough_enabled(general_settings: Mapping[str, object]) -> bool:
|
||||
setting: Final = general_settings.get("enable_openai_websocket_passthrough")
|
||||
if isinstance(setting, str):
|
||||
return str_to_bool(setting) is True
|
||||
return setting is True
|
||||
|
||||
|
||||
class _OpenAIWebsocketModelAllowlists(Protocol):
|
||||
async def __call__(self, valid_token: UserAPIKeyAuth, /) -> tuple[Sequence[str], ...]: ...
|
||||
|
||||
|
||||
async def _openai_websocket_refusal(
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
general_settings: Mapping[str, object],
|
||||
model_allowlists: _OpenAIWebsocketModelAllowlists,
|
||||
) -> _OpenAIWebsocketRefusal | None:
|
||||
if not _is_openai_websocket_passthrough_enabled(general_settings):
|
||||
return _OPENAI_WS_DISABLED_REFUSAL
|
||||
if _has_model_restrictions(await model_allowlists(user_api_key_dict)):
|
||||
return _OPENAI_WS_MODEL_RESTRICTED_REFUSAL
|
||||
return None
|
||||
|
||||
|
||||
class _OpenAIWebsocketRelay(Protocol):
|
||||
async def __call__(
|
||||
self,
|
||||
*,
|
||||
websocket: WebSocket,
|
||||
target: str,
|
||||
custom_headers: dict[str, str], # mutable-ok: the relay takes a plain dict of upstream headers
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
forward_headers: bool,
|
||||
endpoint: str,
|
||||
accept_websocket: bool,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
def _proxy_general_settings() -> Mapping[str, object]:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
return general_settings
|
||||
|
||||
|
||||
def _openai_websocket_relay() -> _OpenAIWebsocketRelay:
|
||||
return websocket_passthrough_request
|
||||
|
||||
|
||||
def _proxy_model_allowlists() -> _OpenAIWebsocketModelAllowlists:
|
||||
from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
|
||||
|
||||
async def resolve(valid_token: UserAPIKeyAuth, /) -> tuple[Sequence[str], ...]:
|
||||
return await enforced_model_allowlists(
|
||||
valid_token=valid_token,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
return resolve
|
||||
|
||||
|
||||
@router.websocket("/openai_passthrough/{endpoint:path}")
|
||||
|
|
@ -2351,13 +2447,27 @@ async def openai_websocket_proxy_route(
|
|||
websocket: WebSocket,
|
||||
endpoint: str,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)],
|
||||
general_settings: Annotated[Mapping[str, object], Depends(_proxy_general_settings)],
|
||||
relay: Annotated[_OpenAIWebsocketRelay, Depends(_openai_websocket_relay)],
|
||||
model_allowlists: Annotated[_OpenAIWebsocketModelAllowlists, Depends(_proxy_model_allowlists)],
|
||||
) -> None:
|
||||
"""WebSocket passthrough for OpenAI prefixes (realtime / responses.connect)."""
|
||||
if _key_has_model_restrictions(user_api_key_dict):
|
||||
await websocket.close(
|
||||
code=1008,
|
||||
reason="Keys with model restrictions cannot use OpenAI websocket passthrough",
|
||||
)
|
||||
requested_subprotocols: Final = tuple(
|
||||
protocol.strip()
|
||||
for protocol in (websocket.headers.get("sec-websocket-protocol") or "").split(",")
|
||||
if protocol.strip()
|
||||
)
|
||||
negotiated_subprotocol: Final = requested_subprotocols[0] if requested_subprotocols else None
|
||||
|
||||
refusal: Final = await _openai_websocket_refusal(user_api_key_dict, general_settings, model_allowlists)
|
||||
if refusal is not None:
|
||||
await websocket.accept(subprotocol=negotiated_subprotocol)
|
||||
error_frame: Final[_OpenAIWebsocketErrorFrame] = {
|
||||
"type": "error",
|
||||
"error": {"type": "invalid_request_error", "message": refusal.message},
|
||||
}
|
||||
await websocket.send_text(json.dumps(error_frame))
|
||||
await websocket.close(code=1008, reason=refusal.close_reason)
|
||||
return
|
||||
|
||||
base_target_url: Final = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/"
|
||||
|
|
@ -2393,14 +2503,9 @@ async def openai_websocket_proxy_route(
|
|||
"Authorization": f"Bearer {openai_api_key}"
|
||||
}
|
||||
|
||||
requested_subprotocols: Final = tuple(
|
||||
protocol.strip()
|
||||
for protocol in (websocket.headers.get("sec-websocket-protocol") or "").split(",")
|
||||
if protocol.strip()
|
||||
)
|
||||
await websocket.accept(subprotocol=requested_subprotocols[0] if requested_subprotocols else None)
|
||||
await websocket.accept(subprotocol=negotiated_subprotocol)
|
||||
|
||||
await websocket_passthrough_request(
|
||||
await relay(
|
||||
websocket=websocket,
|
||||
target=wss_target,
|
||||
custom_headers=custom_headers,
|
||||
|
|
|
|||
|
|
@ -6810,6 +6810,11 @@ class ProxyConfig:
|
|||
else:
|
||||
general_settings["apply_user_budget_to_team_keys"] = db_value if db_value is None else bool(db_value)
|
||||
|
||||
if "enable_openai_websocket_passthrough" not in self._yaml_general_settings_keys:
|
||||
general_settings["enable_openai_websocket_passthrough"] = _general_settings.get(
|
||||
"enable_openai_websocket_passthrough"
|
||||
)
|
||||
|
||||
## STORE MODEL IN DB ##
|
||||
if "store_model_in_db" in _general_settings:
|
||||
value = _general_settings["store_model_in_db"]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"limit": 733
|
||||
},
|
||||
"TQ002": {
|
||||
"limit": 741
|
||||
"limit": 737
|
||||
},
|
||||
"TQ003": {
|
||||
"limit": 62
|
||||
|
|
@ -21,6 +21,6 @@
|
|||
"limit": 117
|
||||
},
|
||||
"TQ008": {
|
||||
"limit": 11003
|
||||
"limit": 10993
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,16 +1,39 @@
|
|||
"""OpenAI passthrough must register WebSocket catch-all routes (#36088)."""
|
||||
"""OpenAI passthrough WebSocket route: registration, opt-in gating, and refusals."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from starlette.routing import WebSocketRoute
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
_OPENAI_WS_DISABLED_REFUSAL,
|
||||
_OPENAI_WS_MODEL_RESTRICTED_REFUSAL,
|
||||
_has_model_restrictions,
|
||||
_openai_websocket_refusal,
|
||||
_proxy_model_allowlists,
|
||||
openai_websocket_proxy_route,
|
||||
router,
|
||||
)
|
||||
|
||||
Scopes = tuple[Sequence[str], ...]
|
||||
|
||||
ENABLED: Final = MappingProxyType({"enable_openai_websocket_passthrough": True})
|
||||
DISABLED_SETTINGS: Final = (
|
||||
MappingProxyType({}),
|
||||
MappingProxyType({"enable_openai_websocket_passthrough": False}),
|
||||
MappingProxyType({"enable_openai_websocket_passthrough": "false"}),
|
||||
MappingProxyType({"enable_openai_websocket_passthrough": None}),
|
||||
)
|
||||
GET_CREDENTIALS: Final = (
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials"
|
||||
)
|
||||
|
||||
|
||||
def test_openai_websocket_passthrough_routes_registered():
|
||||
ws_paths = {route.path for route in router.routes if isinstance(route, WebSocketRoute)}
|
||||
|
|
@ -18,164 +41,258 @@ def test_openai_websocket_passthrough_routes_registered():
|
|||
assert "/openai_passthrough/{endpoint:path}" in ws_paths
|
||||
|
||||
|
||||
def _mock_websocket(path: str, query: str, headers: dict[str, str] | None = None) -> MagicMock:
|
||||
websocket = MagicMock()
|
||||
websocket.url.path = path
|
||||
websocket.url.query = query
|
||||
websocket.headers = headers or {}
|
||||
websocket.accept = AsyncMock()
|
||||
websocket.close = AsyncMock()
|
||||
return websocket
|
||||
class _FakeWebSocket:
|
||||
def __init__(self, path: str, query: str, subprotocols: str | None = None) -> None:
|
||||
self.url = SimpleNamespace(path=path, query=query)
|
||||
self.headers = {"sec-websocket-protocol": subprotocols} if subprotocols else {}
|
||||
self.accepts: list[str | None] = []
|
||||
self.sent: list[str] = []
|
||||
self.closed: tuple[int, str] | None = None
|
||||
|
||||
async def accept(self, subprotocol: str | None = None) -> None:
|
||||
self.accepts.append(subprotocol)
|
||||
|
||||
async def send_text(self, data: str) -> None:
|
||||
self.sent.append(data)
|
||||
|
||||
async def close(self, code: int = 1000, reason: str = "") -> None:
|
||||
self.closed = (code, reason)
|
||||
|
||||
def error_message(self) -> str:
|
||||
assert len(self.sent) == 1
|
||||
frame = json.loads(self.sent[0])
|
||||
assert frame["type"] == "error"
|
||||
return frame["error"]["message"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RelayCall:
|
||||
target: str
|
||||
custom_headers: Mapping[str, str]
|
||||
forward_headers: bool
|
||||
endpoint: str
|
||||
accept_websocket: bool
|
||||
|
||||
|
||||
class _FakeRelay:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[_RelayCall] = []
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
*,
|
||||
websocket: _FakeWebSocket,
|
||||
target: str,
|
||||
custom_headers: dict[str, str],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
forward_headers: bool,
|
||||
endpoint: str,
|
||||
accept_websocket: bool,
|
||||
) -> None:
|
||||
self.calls.append(
|
||||
_RelayCall(
|
||||
target=target,
|
||||
custom_headers=MappingProxyType(dict(custom_headers)),
|
||||
forward_headers=forward_headers,
|
||||
endpoint=endpoint,
|
||||
accept_websocket=accept_websocket,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _FakeModelAllowlists:
|
||||
def __init__(self, scopes: Scopes) -> None:
|
||||
self.scopes = scopes
|
||||
self.calls: list[UserAPIKeyAuth] = []
|
||||
|
||||
async def __call__(self, valid_token: UserAPIKeyAuth, /) -> Scopes:
|
||||
self.calls.append(valid_token)
|
||||
return self.scopes
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Served:
|
||||
relay: _FakeRelay
|
||||
allowlists: _FakeModelAllowlists
|
||||
|
||||
|
||||
async def _serve(
|
||||
websocket: _FakeWebSocket,
|
||||
endpoint: str,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
general_settings: Mapping[str, object],
|
||||
scopes: Scopes = (),
|
||||
) -> _Served:
|
||||
served = _Served(relay=_FakeRelay(), allowlists=_FakeModelAllowlists(scopes))
|
||||
await openai_websocket_proxy_route(
|
||||
websocket=websocket,
|
||||
endpoint=endpoint,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
general_settings=general_settings,
|
||||
relay=served.relay,
|
||||
model_allowlists=served.allowlists,
|
||||
)
|
||||
return served
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prefix", ["openai", "openai_passthrough"])
|
||||
async def test_openai_websocket_forwards_query_and_keeps_provider_auth(prefix):
|
||||
websocket = _mock_websocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview")
|
||||
async def test_openai_websocket_forwards_query_and_keeps_provider_auth(prefix, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
websocket = _FakeWebSocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials",
|
||||
return_value="sk-provider",
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._join_url_paths",
|
||||
return_value="https://api.openai.com/v1/realtime",
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws,
|
||||
):
|
||||
await openai_websocket_proxy_route(
|
||||
websocket=websocket,
|
||||
endpoint="v1/realtime",
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
with patch(GET_CREDENTIALS, return_value="sk-provider"):
|
||||
served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED)
|
||||
|
||||
assert served.relay.calls == [
|
||||
_RelayCall(
|
||||
target="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview",
|
||||
custom_headers=MappingProxyType({"Authorization": "Bearer sk-provider"}),
|
||||
forward_headers=False,
|
||||
endpoint=f"/{prefix}/v1/realtime",
|
||||
accept_websocket=False,
|
||||
)
|
||||
|
||||
kwargs = mock_ws.await_args.kwargs
|
||||
assert kwargs["target"] == "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"
|
||||
assert kwargs["custom_headers"] == {"Authorization": "Bearer sk-provider"}
|
||||
assert kwargs["forward_headers"] is False
|
||||
assert kwargs["endpoint"] == f"/{prefix}/v1/realtime"
|
||||
assert kwargs["accept_websocket"] is False
|
||||
websocket.accept.assert_awaited_once_with(subprotocol=None)
|
||||
websocket.close.assert_not_awaited()
|
||||
]
|
||||
assert websocket.accepts == [None]
|
||||
assert websocket.sent == []
|
||||
assert websocket.closed is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_websocket_accepts_first_client_subprotocol():
|
||||
websocket = _mock_websocket(
|
||||
websocket = _FakeWebSocket(
|
||||
"/openai/v1/realtime",
|
||||
"model=gpt-4o-realtime-preview",
|
||||
headers={
|
||||
"sec-websocket-protocol": "realtime, openai-insecure-api-key.sk-abc, openai-beta.realtime-v1"
|
||||
},
|
||||
subprotocols="realtime, openai-insecure-api-key.sk-abc, openai-beta.realtime-v1",
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials",
|
||||
return_value="sk-provider",
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws,
|
||||
):
|
||||
await openai_websocket_proxy_route(
|
||||
websocket=websocket,
|
||||
endpoint="v1/realtime",
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
with patch(GET_CREDENTIALS, return_value="sk-provider"):
|
||||
served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED)
|
||||
|
||||
websocket.accept.assert_awaited_once_with(subprotocol="realtime")
|
||||
assert mock_ws.await_args.kwargs["accept_websocket"] is False
|
||||
websocket.close.assert_not_awaited()
|
||||
assert websocket.accepts == ["realtime"]
|
||||
assert [call.accept_websocket for call in served.relay.calls] == [False]
|
||||
assert websocket.closed is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_websocket_closes_cleanly_when_provider_credentials_missing():
|
||||
websocket = _mock_websocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview")
|
||||
websocket = _FakeWebSocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws,
|
||||
):
|
||||
await openai_websocket_proxy_route(
|
||||
websocket=websocket,
|
||||
endpoint="v1/realtime",
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
with patch(GET_CREDENTIALS, return_value=None):
|
||||
served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED)
|
||||
|
||||
websocket.close.assert_awaited_once()
|
||||
assert websocket.close.await_args.kwargs["code"] == 1011
|
||||
websocket.accept.assert_not_awaited()
|
||||
mock_ws.assert_not_awaited()
|
||||
assert websocket.closed is not None
|
||||
assert websocket.closed[0] == 1011
|
||||
assert "OPENAI_API_KEY" in websocket.closed[1]
|
||||
assert websocket.accepts == []
|
||||
assert served.relay.calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"user_api_key_dict",
|
||||
[
|
||||
UserAPIKeyAuth(models=["gpt-4o"]),
|
||||
UserAPIKeyAuth(team_models=["gpt-4o-realtime-preview"]),
|
||||
UserAPIKeyAuth(models=["all-team-models"], team_models=["gpt-4o"]),
|
||||
],
|
||||
)
|
||||
async def test_openai_websocket_rejects_model_restricted_keys(user_api_key_dict):
|
||||
websocket = _mock_websocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview")
|
||||
@pytest.mark.parametrize("prefix", ["openai", "openai_passthrough"])
|
||||
@pytest.mark.parametrize("general_settings", DISABLED_SETTINGS)
|
||||
async def test_openai_websocket_refused_unless_explicitly_enabled(prefix, general_settings):
|
||||
websocket = _FakeWebSocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview")
|
||||
|
||||
with patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await openai_websocket_proxy_route(
|
||||
websocket=websocket,
|
||||
endpoint="v1/realtime",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), general_settings)
|
||||
|
||||
websocket.close.assert_awaited_once()
|
||||
assert websocket.close.await_args.kwargs["code"] == 1008
|
||||
websocket.accept.assert_not_awaited()
|
||||
mock_ws.assert_not_awaited()
|
||||
assert "enable_openai_websocket_passthrough" in websocket.error_message()
|
||||
assert websocket.accepts == [None]
|
||||
assert websocket.closed == (1008, _OPENAI_WS_DISABLED_REFUSAL.close_reason)
|
||||
assert served.relay.calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"user_api_key_dict",
|
||||
[
|
||||
UserAPIKeyAuth(),
|
||||
UserAPIKeyAuth(models=["all-proxy-models"]),
|
||||
UserAPIKeyAuth(models=["*"]),
|
||||
UserAPIKeyAuth(models=["all-team-models"], team_models=["all-proxy-models"]),
|
||||
],
|
||||
@pytest.mark.parametrize("general_settings", DISABLED_SETTINGS)
|
||||
async def test_openai_websocket_refusal_is_disabled_for_falsy_settings(general_settings):
|
||||
refusal = await _openai_websocket_refusal(UserAPIKeyAuth(), general_settings, _FakeModelAllowlists(()))
|
||||
assert refusal is _OPENAI_WS_DISABLED_REFUSAL
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("value", [True, "true", "True"])
|
||||
async def test_openai_websocket_refusal_is_none_for_truthy_settings(value):
|
||||
settings = MappingProxyType({"enable_openai_websocket_passthrough": value})
|
||||
assert await _openai_websocket_refusal(UserAPIKeyAuth(), settings, _FakeModelAllowlists(())) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_websocket_refusal_echoes_requested_subprotocol():
|
||||
websocket = _FakeWebSocket(
|
||||
"/openai_passthrough/v1/realtime",
|
||||
"model=gpt-4o-realtime-preview",
|
||||
subprotocols="realtime, openai-beta.realtime-v1",
|
||||
)
|
||||
|
||||
served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), MappingProxyType({}))
|
||||
|
||||
assert websocket.accepts == ["realtime"]
|
||||
assert websocket.closed == (1008, _OPENAI_WS_DISABLED_REFUSAL.close_reason)
|
||||
assert served.relay.calls == []
|
||||
|
||||
|
||||
RESTRICTED_SCOPES: Final[tuple[Scopes, ...]] = (
|
||||
(("gpt-4o",),),
|
||||
((), ("gpt-4o-realtime-preview",)),
|
||||
(("all-team-models",), ("gpt-4o",)),
|
||||
((), ("all-proxy-models",), ("gpt-4o",)),
|
||||
((), (), (), ("gpt-4o",)),
|
||||
(("*",), (), (), (), ("gpt-4o",)),
|
||||
)
|
||||
UNRESTRICTED_SCOPES: Final[tuple[Scopes, ...]] = (
|
||||
(),
|
||||
((),),
|
||||
(("all-proxy-models",),),
|
||||
(("*",),),
|
||||
(("all-team-models",), ("all-proxy-models",)),
|
||||
((), (), (), (), ()),
|
||||
(("*",), ("all-proxy-models",), ("all-team-models",), (), ()),
|
||||
)
|
||||
async def test_openai_websocket_allows_unrestricted_keys(user_api_key_dict):
|
||||
websocket = _mock_websocket("/openai/v1/responses", "")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials",
|
||||
return_value="sk-provider",
|
||||
),
|
||||
patch(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws,
|
||||
):
|
||||
await openai_websocket_proxy_route(
|
||||
websocket=websocket,
|
||||
endpoint="v1/responses",
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
mock_ws.assert_awaited_once()
|
||||
websocket.close.assert_not_awaited()
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("scopes", RESTRICTED_SCOPES)
|
||||
async def test_openai_websocket_rejects_model_restricted_identities(scopes):
|
||||
websocket = _FakeWebSocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview")
|
||||
user_api_key_dict = UserAPIKeyAuth(token="hashed-fake", user_id="user-fake", team_id="team-fake")
|
||||
|
||||
served = await _serve(websocket, "v1/realtime", user_api_key_dict, ENABLED, scopes)
|
||||
|
||||
assert "model restrictions" in websocket.error_message()
|
||||
assert websocket.closed == (1008, _OPENAI_WS_MODEL_RESTRICTED_REFUSAL.close_reason)
|
||||
assert served.relay.calls == []
|
||||
assert served.allowlists.calls == [user_api_key_dict]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("scopes", RESTRICTED_SCOPES)
|
||||
async def test_openai_websocket_disabled_refusal_skips_allowlist_lookups(scopes):
|
||||
allowlists = _FakeModelAllowlists(scopes)
|
||||
|
||||
refusal = await _openai_websocket_refusal(UserAPIKeyAuth(), MappingProxyType({}), allowlists)
|
||||
|
||||
assert refusal is _OPENAI_WS_DISABLED_REFUSAL
|
||||
assert allowlists.calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("scopes", UNRESTRICTED_SCOPES)
|
||||
async def test_openai_websocket_allows_unrestricted_identities(scopes):
|
||||
websocket = _FakeWebSocket("/openai/v1/responses", "")
|
||||
|
||||
with patch(GET_CREDENTIALS, return_value="sk-provider"):
|
||||
served = await _serve(websocket, "v1/responses", UserAPIKeyAuth(), ENABLED, scopes)
|
||||
|
||||
assert len(served.relay.calls) == 1
|
||||
assert websocket.sent == []
|
||||
assert websocket.closed is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_model_allowlists_reads_the_token_scopes_without_a_database():
|
||||
token: Final = UserAPIKeyAuth(models=[], team_id="team-fake", team_models=["gpt-4o"])
|
||||
with patch("litellm.proxy.proxy_server.prisma_client", None):
|
||||
scopes = await _proxy_model_allowlists()(token)
|
||||
|
||||
assert tuple(tuple(scope) for scope in scopes) == ((), ("gpt-4o",))
|
||||
assert _has_model_restrictions(scopes)
|
||||
|
|
|
|||
|
|
@ -11568,14 +11568,10 @@ async def test_key_window_spend_row_is_enqueued_with_the_actual_cost():
|
|||
|
||||
reset_at = datetime.now(timezone.utc) + timedelta(days=10)
|
||||
key_obj = MagicMock()
|
||||
key_obj.budget_limits = [
|
||||
{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}
|
||||
]
|
||||
key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}]
|
||||
|
||||
with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue:
|
||||
await increment_spend_counters(
|
||||
token="hashed-token", team_id=None, user_id=None, response_cost=0.25
|
||||
)
|
||||
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
|
||||
enqueued = await _drain(queue)
|
||||
|
||||
assert len(enqueued) == 1
|
||||
|
|
@ -11594,14 +11590,10 @@ async def test_team_window_spend_row_is_enqueued():
|
|||
|
||||
reset_at = datetime.now(timezone.utc) + timedelta(days=3)
|
||||
team_obj = MagicMock()
|
||||
team_obj.budget_limits = [
|
||||
{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}
|
||||
]
|
||||
team_obj.budget_limits = [{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}]
|
||||
|
||||
with _window_spend_enqueue_env({"team_id:team-1": team_obj}) as queue:
|
||||
await increment_spend_counters(
|
||||
token=None, team_id="team-1", user_id=None, response_cost=1.5
|
||||
)
|
||||
await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5)
|
||||
enqueued = await _drain(queue)
|
||||
|
||||
assert len(enqueued) == 1
|
||||
|
|
@ -11620,9 +11612,7 @@ async def test_window_spend_row_is_enqueued_even_when_the_counter_was_reserved()
|
|||
|
||||
reset_at = datetime.now(timezone.utc) + timedelta(days=10)
|
||||
key_obj = MagicMock()
|
||||
key_obj.budget_limits = [
|
||||
{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}
|
||||
]
|
||||
key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}]
|
||||
reservation = {
|
||||
"entries": [
|
||||
{"counter_key": "spend:key:hashed-token", "reserved": 1.0},
|
||||
|
|
@ -11660,9 +11650,7 @@ async def test_sliding_window_without_reset_at_is_not_enqueued():
|
|||
key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0}]
|
||||
|
||||
with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue:
|
||||
await increment_spend_counters(
|
||||
token="hashed-token", team_id=None, user_id=None, response_cost=0.25
|
||||
)
|
||||
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
|
||||
enqueued = await _drain(queue)
|
||||
|
||||
assert enqueued == []
|
||||
|
|
@ -11680,9 +11668,7 @@ async def test_each_configured_window_gets_its_own_row_enqueue():
|
|||
]
|
||||
|
||||
with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue:
|
||||
await increment_spend_counters(
|
||||
token="hashed-token", team_id=None, user_id=None, response_cost=0.25
|
||||
)
|
||||
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
|
||||
enqueued = await _drain(queue)
|
||||
|
||||
assert sorted(item["window_duration"] for item in enqueued) == ["1d", "30d"]
|
||||
|
|
@ -11697,9 +11683,7 @@ async def test_no_window_spend_row_enqueued_without_budget_limits():
|
|||
key_obj.budget_limits = None
|
||||
|
||||
with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue:
|
||||
await increment_spend_counters(
|
||||
token="hashed-token", team_id=None, user_id=None, response_cost=0.25
|
||||
)
|
||||
await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25)
|
||||
enqueued = await _drain(queue)
|
||||
|
||||
assert enqueued == []
|
||||
|
|
@ -11713,9 +11697,7 @@ async def test_window_spend_row_carries_the_request_start_time():
|
|||
|
||||
reset_at = datetime.now(timezone.utc) + timedelta(days=10)
|
||||
key_obj = MagicMock()
|
||||
key_obj.budget_limits = [
|
||||
{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}
|
||||
]
|
||||
key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}]
|
||||
|
||||
with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue:
|
||||
await increment_spend_counters(
|
||||
|
|
@ -11736,9 +11718,7 @@ async def test_team_window_spend_row_carries_the_request_start_time():
|
|||
|
||||
reset_at = datetime.now(timezone.utc) + timedelta(days=3)
|
||||
team_obj = MagicMock()
|
||||
team_obj.budget_limits = [
|
||||
{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}
|
||||
]
|
||||
team_obj.budget_limits = [{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}]
|
||||
|
||||
with _window_spend_enqueue_env({"team_id:team-1": team_obj}) as queue:
|
||||
await increment_spend_counters(
|
||||
|
|
@ -12050,7 +12030,6 @@ async def test_init_guardrails_in_db_snapshots_and_reconciles_under_guardrail_re
|
|||
assert not GUARDRAIL_RECONCILE_LOCK.locked()
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeypatch):
|
||||
from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY
|
||||
|
|
@ -12094,7 +12073,9 @@ async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeyp
|
|||
await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client)
|
||||
assert served_content() == "Begin every reply with AHOY"
|
||||
|
||||
prisma_client.db.litellm_prompttable.find_many = AsyncMock(return_value=[db_row("Begin every reply with HOWDY")])
|
||||
prisma_client.db.litellm_prompttable.find_many = AsyncMock(
|
||||
return_value=[db_row("Begin every reply with HOWDY")]
|
||||
)
|
||||
await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client)
|
||||
|
||||
assert served_content() == "Begin every reply with HOWDY"
|
||||
|
|
@ -12547,3 +12528,40 @@ def test_disabling_docs_does_not_disable_other_routes(monkeypatch):
|
|||
|
||||
assert client.get("/redoc").status_code == 404
|
||||
assert client.get("/health/liveliness").status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"db_general_settings, expected",
|
||||
[
|
||||
({"enable_openai_websocket_passthrough": True}, True),
|
||||
({"enable_openai_websocket_passthrough": False}, False),
|
||||
({}, None),
|
||||
],
|
||||
)
|
||||
async def test_update_general_settings_propagates_openai_websocket_passthrough(db_general_settings, expected):
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
proxy_config = ProxyConfig()
|
||||
|
||||
with patch("litellm.proxy.proxy_server.general_settings", {"enable_openai_websocket_passthrough": True}):
|
||||
await proxy_config._update_general_settings(db_general_settings=db_general_settings)
|
||||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
||||
assert ps.general_settings["enable_openai_websocket_passthrough"] is expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_general_settings_keeps_yaml_openai_websocket_passthrough():
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
proxy_config = ProxyConfig()
|
||||
proxy_config._yaml_general_settings_keys = {"enable_openai_websocket_passthrough"}
|
||||
|
||||
with patch("litellm.proxy.proxy_server.general_settings", {"enable_openai_websocket_passthrough": False}):
|
||||
await proxy_config._update_general_settings(db_general_settings={"enable_openai_websocket_passthrough": True})
|
||||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
||||
assert ps.general_settings["enable_openai_websocket_passthrough"] is False
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -25776,6 +25776,11 @@ export interface components {
|
|||
* @description If True and SSO is configured (MICROSOFT_CLIENT_ID, GOOGLE_CLIENT_ID, GENERIC_CLIENT_ID, or SAML_IDP_METADATA_URL/XML), disables username/password login on /login, /v2/login, and /v3/login so SSO is the only way to reach the Admin UI. An admin locked out of the UI can still administer the proxy over the API with the master key; unset this setting and restart the proxy to restore UI username/password login. Default is False.
|
||||
*/
|
||||
disable_password_login_when_sso_enabled?: boolean | null;
|
||||
/**
|
||||
* Enable Openai Websocket Passthrough
|
||||
* @description Serve the OpenAI pass-through WebSocket route, which relays frames to OpenAI under the proxy's own provider credential without reading them. Off by default.
|
||||
*/
|
||||
enable_openai_websocket_passthrough?: boolean | null;
|
||||
/**
|
||||
* Enable Public Model Hub
|
||||
* @description Public model hub for users to see what models they have access to, supported openai params, etc.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue