From a2d5215a4fd7e8eb9ff4d2112545bef798b0e86e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:14:44 -0700 Subject: [PATCH 1/9] fix(proxy): gate the OpenAI websocket passthrough behind an explicit opt-in --- litellm/proxy/_types.py | 4 + .../llm_passthrough_endpoints.py | 100 +++++- test-quality-budget.json | 4 +- .../test_openai_ws_passthrough_routes.py | 326 +++++++++++------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 5 files changed, 296 insertions(+), 143 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b33e2fe7ff6..f83011835fd 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2638,6 +2638,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): default=None, description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through", ) + enable_openai_websocket_passthrough: bool | None = Field( + default=None, + description="Serve the OpenAI pass-through WebSocket route, which relays frames to OpenAI under the proxy's own provider credential without reading them. Off by default.", + ) user_header_name: str | None = Field( None, description="[DEPRECATED] Use 'user_header_mappings' instead. When set, the header value is treated as the end user id unless overridden by user_header_mappings.", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 6b1d6405a6a..97d25e20939 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -14,8 +14,9 @@ import json import os import re from collections.abc import AsyncGenerator, Callable, Mapping +from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, cast +from typing import TYPE_CHECKING, Annotated, Final, Protocol, cast import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket @@ -2345,19 +2346,99 @@ def _key_has_model_restrictions(user_api_key_dict: UserAPIKeyAuth) -> bool: return any(str(model) not in _OPENAI_WS_ALL_MODEL_ACCESS for model in scoped_models) +@dataclass(frozen=True, slots=True) +class _OpenAIWebsocketRefusal: + close_reason: str + message: str + + +_OPENAI_WS_DISABLED_REFUSAL: Final = _OpenAIWebsocketRefusal( + close_reason="OpenAI websocket passthrough is disabled", + message=( + "OpenAI websocket passthrough is disabled on this gateway. A proxy admin can turn it on by " + "setting general_settings.enable_openai_websocket_passthrough to true." + ), +) + +_OPENAI_WS_MODEL_RESTRICTED_REFUSAL: Final = _OpenAIWebsocketRefusal( + close_reason="Keys with model restrictions cannot use OpenAI websocket passthrough", + message=( + "Keys with model restrictions cannot use OpenAI websocket passthrough, because this route " + "relays frames to the provider without reading which model they ask for." + ), +) + + +def _is_openai_websocket_passthrough_enabled(general_settings: Mapping[str, object]) -> bool: + setting: Final = general_settings.get("enable_openai_websocket_passthrough") + if isinstance(setting, str): + return str_to_bool(setting) is True + return setting is True + + +def _openai_websocket_refusal( + user_api_key_dict: UserAPIKeyAuth, general_settings: Mapping[str, object] +) -> _OpenAIWebsocketRefusal | None: + if not _is_openai_websocket_passthrough_enabled(general_settings): + return _OPENAI_WS_DISABLED_REFUSAL + if _key_has_model_restrictions(user_api_key_dict): + return _OPENAI_WS_MODEL_RESTRICTED_REFUSAL + return None + + +class _OpenAIWebsocketRelay(Protocol): + async def __call__( + self, + *, + websocket: WebSocket, + target: str, + custom_headers: dict[str, str], # mutable-ok: the relay takes a plain dict of upstream headers + user_api_key_dict: UserAPIKeyAuth, + forward_headers: bool, + endpoint: str, + accept_websocket: bool, + ) -> None: ... + + +def _proxy_general_settings() -> Mapping[str, object]: + from litellm.proxy.proxy_server import general_settings + + return general_settings + + +def _openai_websocket_relay() -> _OpenAIWebsocketRelay: + return websocket_passthrough_request + + @router.websocket("/openai_passthrough/{endpoint:path}") @router.websocket("/openai/{endpoint:path}") async def openai_websocket_proxy_route( websocket: WebSocket, endpoint: str, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)], + general_settings: Annotated[Mapping[str, object], Depends(_proxy_general_settings)], + relay: Annotated[_OpenAIWebsocketRelay, Depends(_openai_websocket_relay)], ) -> None: """WebSocket passthrough for OpenAI prefixes (realtime / responses.connect).""" - if _key_has_model_restrictions(user_api_key_dict): - await websocket.close( - code=1008, - reason="Keys with model restrictions cannot use OpenAI websocket passthrough", + requested_subprotocols: Final = tuple( + protocol.strip() + for protocol in (websocket.headers.get("sec-websocket-protocol") or "").split(",") + if protocol.strip() + ) + negotiated_subprotocol: Final = requested_subprotocols[0] if requested_subprotocols else None + + refusal: Final = _openai_websocket_refusal(user_api_key_dict, general_settings) + if refusal is not None: + await websocket.accept(subprotocol=negotiated_subprotocol) + await websocket.send_text( + json.dumps( + { + "type": "error", + "error": {"type": "invalid_request_error", "message": refusal.message}, + } + ) ) + await websocket.close(code=1008, reason=refusal.close_reason) return base_target_url: Final = os.getenv("OPENAI_API_BASE") or "https://api.openai.com/" @@ -2393,14 +2474,9 @@ async def openai_websocket_proxy_route( "Authorization": f"Bearer {openai_api_key}" } - requested_subprotocols: Final = tuple( - protocol.strip() - for protocol in (websocket.headers.get("sec-websocket-protocol") or "").split(",") - if protocol.strip() - ) - await websocket.accept(subprotocol=requested_subprotocols[0] if requested_subprotocols else None) + await websocket.accept(subprotocol=negotiated_subprotocol) - await websocket_passthrough_request( + await relay( websocket=websocket, target=wss_target, custom_headers=custom_headers, diff --git a/test-quality-budget.json b/test-quality-budget.json index 7ca563d25af..3c12371f02f 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -3,7 +3,7 @@ "limit": 733 }, "TQ002": { - "limit": 741 + "limit": 737 }, "TQ003": { "limit": 62 @@ -21,6 +21,6 @@ "limit": 117 }, "TQ008": { - "limit": 11003 + "limit": 10993 } } diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py index b22e202d9e0..6578b75ace1 100644 --- a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -1,16 +1,35 @@ -"""OpenAI passthrough must register WebSocket catch-all routes (#36088).""" +"""OpenAI passthrough WebSocket route: registration, opt-in gating, and refusals.""" -from unittest.mock import AsyncMock, MagicMock, patch +import json +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType, SimpleNamespace +from typing import Final +from unittest.mock import patch import pytest from starlette.routing import WebSocketRoute from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + _OPENAI_WS_DISABLED_REFUSAL, + _OPENAI_WS_MODEL_RESTRICTED_REFUSAL, + _openai_websocket_refusal, openai_websocket_proxy_route, router, ) +ENABLED: Final = MappingProxyType({"enable_openai_websocket_passthrough": True}) +DISABLED_SETTINGS: Final = ( + MappingProxyType({}), + MappingProxyType({"enable_openai_websocket_passthrough": False}), + MappingProxyType({"enable_openai_websocket_passthrough": "false"}), + MappingProxyType({"enable_openai_websocket_passthrough": None}), +) +GET_CREDENTIALS: Final = ( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials" +) + def test_openai_websocket_passthrough_routes_registered(): ws_paths = {route.path for route in router.routes if isinstance(route, WebSocketRoute)} @@ -18,164 +37,213 @@ def test_openai_websocket_passthrough_routes_registered(): assert "/openai_passthrough/{endpoint:path}" in ws_paths -def _mock_websocket(path: str, query: str, headers: dict[str, str] | None = None) -> MagicMock: - websocket = MagicMock() - websocket.url.path = path - websocket.url.query = query - websocket.headers = headers or {} - websocket.accept = AsyncMock() - websocket.close = AsyncMock() - return websocket +class _FakeWebSocket: + def __init__(self, path: str, query: str, subprotocols: str | None = None) -> None: + self.url = SimpleNamespace(path=path, query=query) + self.headers = {"sec-websocket-protocol": subprotocols} if subprotocols else {} + self.accepts: list[str | None] = [] + self.sent: list[str] = [] + self.closed: tuple[int, str] | None = None + + async def accept(self, subprotocol: str | None = None) -> None: + self.accepts.append(subprotocol) + + async def send_text(self, data: str) -> None: + self.sent.append(data) + + async def close(self, code: int = 1000, reason: str = "") -> None: + self.closed = (code, reason) + + def error_message(self) -> str: + assert len(self.sent) == 1 + frame = json.loads(self.sent[0]) + assert frame["type"] == "error" + return frame["error"]["message"] + + +@dataclass(frozen=True, slots=True) +class _RelayCall: + target: str + custom_headers: Mapping[str, str] + forward_headers: bool + endpoint: str + accept_websocket: bool + + +class _FakeRelay: + def __init__(self) -> None: + self.calls: list[_RelayCall] = [] + + async def __call__( + self, + *, + websocket: _FakeWebSocket, + target: str, + custom_headers: dict[str, str], + user_api_key_dict: UserAPIKeyAuth, + forward_headers: bool, + endpoint: str, + accept_websocket: bool, + ) -> None: + self.calls.append( + _RelayCall( + target=target, + custom_headers=MappingProxyType(dict(custom_headers)), + forward_headers=forward_headers, + endpoint=endpoint, + accept_websocket=accept_websocket, + ) + ) + + +async def _serve( + websocket: _FakeWebSocket, + endpoint: str, + user_api_key_dict: UserAPIKeyAuth, + general_settings: Mapping[str, object], +) -> _FakeRelay: + relay = _FakeRelay() + await openai_websocket_proxy_route( + websocket=websocket, + endpoint=endpoint, + user_api_key_dict=user_api_key_dict, + general_settings=general_settings, + relay=relay, + ) + return relay @pytest.mark.asyncio @pytest.mark.parametrize("prefix", ["openai", "openai_passthrough"]) -async def test_openai_websocket_forwards_query_and_keeps_provider_auth(prefix): - websocket = _mock_websocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview") +async def test_openai_websocket_forwards_query_and_keeps_provider_auth(prefix, monkeypatch): + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + websocket = _FakeWebSocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview") - with ( - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", - return_value="sk-provider", - ), - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._join_url_paths", - return_value="https://api.openai.com/v1/realtime", - ), - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", - new_callable=AsyncMock, - ) as mock_ws, - ): - await openai_websocket_proxy_route( - websocket=websocket, - endpoint="v1/realtime", - user_api_key_dict=UserAPIKeyAuth(), + with patch(GET_CREDENTIALS, return_value="sk-provider"): + relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) + + assert relay.calls == [ + _RelayCall( + target="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview", + custom_headers=MappingProxyType({"Authorization": "Bearer sk-provider"}), + forward_headers=False, + endpoint=f"/{prefix}/v1/realtime", + accept_websocket=False, ) - - kwargs = mock_ws.await_args.kwargs - assert kwargs["target"] == "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview" - assert kwargs["custom_headers"] == {"Authorization": "Bearer sk-provider"} - assert kwargs["forward_headers"] is False - assert kwargs["endpoint"] == f"/{prefix}/v1/realtime" - assert kwargs["accept_websocket"] is False - websocket.accept.assert_awaited_once_with(subprotocol=None) - websocket.close.assert_not_awaited() + ] + assert websocket.accepts == [None] + assert websocket.sent == [] + assert websocket.closed is None @pytest.mark.asyncio async def test_openai_websocket_accepts_first_client_subprotocol(): - websocket = _mock_websocket( + websocket = _FakeWebSocket( "/openai/v1/realtime", "model=gpt-4o-realtime-preview", - headers={ - "sec-websocket-protocol": "realtime, openai-insecure-api-key.sk-abc, openai-beta.realtime-v1" - }, + subprotocols="realtime, openai-insecure-api-key.sk-abc, openai-beta.realtime-v1", ) - with ( - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", - return_value="sk-provider", - ), - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", - new_callable=AsyncMock, - ) as mock_ws, - ): - await openai_websocket_proxy_route( - websocket=websocket, - endpoint="v1/realtime", - user_api_key_dict=UserAPIKeyAuth(), - ) + with patch(GET_CREDENTIALS, return_value="sk-provider"): + relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) - websocket.accept.assert_awaited_once_with(subprotocol="realtime") - assert mock_ws.await_args.kwargs["accept_websocket"] is False - websocket.close.assert_not_awaited() + assert websocket.accepts == ["realtime"] + assert [call.accept_websocket for call in relay.calls] == [False] + assert websocket.closed is None @pytest.mark.asyncio async def test_openai_websocket_closes_cleanly_when_provider_credentials_missing(): - websocket = _mock_websocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") + websocket = _FakeWebSocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") - with ( - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", - return_value=None, - ), - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", - new_callable=AsyncMock, - ) as mock_ws, - ): - await openai_websocket_proxy_route( - websocket=websocket, - endpoint="v1/realtime", - user_api_key_dict=UserAPIKeyAuth(), - ) + with patch(GET_CREDENTIALS, return_value=None): + relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) - websocket.close.assert_awaited_once() - assert websocket.close.await_args.kwargs["code"] == 1011 - websocket.accept.assert_not_awaited() - mock_ws.assert_not_awaited() + assert websocket.closed is not None + assert websocket.closed[0] == 1011 + assert "OPENAI_API_KEY" in websocket.closed[1] + assert websocket.accepts == [] + assert relay.calls == [] @pytest.mark.asyncio -@pytest.mark.parametrize( - "user_api_key_dict", - [ - UserAPIKeyAuth(models=["gpt-4o"]), - UserAPIKeyAuth(team_models=["gpt-4o-realtime-preview"]), - UserAPIKeyAuth(models=["all-team-models"], team_models=["gpt-4o"]), - ], +@pytest.mark.parametrize("prefix", ["openai", "openai_passthrough"]) +@pytest.mark.parametrize("general_settings", DISABLED_SETTINGS) +async def test_openai_websocket_refused_unless_explicitly_enabled(prefix, general_settings): + websocket = _FakeWebSocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview") + + relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), general_settings) + + assert "enable_openai_websocket_passthrough" in websocket.error_message() + assert websocket.accepts == [None] + assert websocket.closed == (1008, _OPENAI_WS_DISABLED_REFUSAL.close_reason) + assert relay.calls == [] + + +@pytest.mark.parametrize("general_settings", DISABLED_SETTINGS) +def test_openai_websocket_refusal_is_disabled_for_falsy_settings(general_settings): + assert _openai_websocket_refusal(UserAPIKeyAuth(), general_settings) is _OPENAI_WS_DISABLED_REFUSAL + + +@pytest.mark.parametrize("value", [True, "true", "True"]) +def test_openai_websocket_refusal_is_none_for_truthy_settings(value): + settings = MappingProxyType({"enable_openai_websocket_passthrough": value}) + assert _openai_websocket_refusal(UserAPIKeyAuth(), settings) is None + + +@pytest.mark.asyncio +async def test_openai_websocket_refusal_echoes_requested_subprotocol(): + websocket = _FakeWebSocket( + "/openai_passthrough/v1/realtime", + "model=gpt-4o-realtime-preview", + subprotocols="realtime, openai-beta.realtime-v1", + ) + + relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), MappingProxyType({})) + + assert websocket.accepts == ["realtime"] + assert websocket.closed == (1008, _OPENAI_WS_DISABLED_REFUSAL.close_reason) + assert relay.calls == [] + + +RESTRICTED_KEYS: Final = ( + UserAPIKeyAuth(models=["gpt-4o"]), + UserAPIKeyAuth(team_models=["gpt-4o-realtime-preview"]), + UserAPIKeyAuth(models=["all-team-models"], team_models=["gpt-4o"]), ) +UNRESTRICTED_KEYS: Final = ( + UserAPIKeyAuth(), + UserAPIKeyAuth(models=["all-proxy-models"]), + UserAPIKeyAuth(models=["*"]), + UserAPIKeyAuth(models=["all-team-models"], team_models=["all-proxy-models"]), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("user_api_key_dict", RESTRICTED_KEYS) async def test_openai_websocket_rejects_model_restricted_keys(user_api_key_dict): - websocket = _mock_websocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") + websocket = _FakeWebSocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") - with patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", - new_callable=AsyncMock, - ) as mock_ws: - await openai_websocket_proxy_route( - websocket=websocket, - endpoint="v1/realtime", - user_api_key_dict=user_api_key_dict, - ) + relay = await _serve(websocket, "v1/realtime", user_api_key_dict, ENABLED) - websocket.close.assert_awaited_once() - assert websocket.close.await_args.kwargs["code"] == 1008 - websocket.accept.assert_not_awaited() - mock_ws.assert_not_awaited() + assert "model restrictions" in websocket.error_message() + assert websocket.closed == (1008, _OPENAI_WS_MODEL_RESTRICTED_REFUSAL.close_reason) + assert relay.calls == [] + + +@pytest.mark.parametrize("user_api_key_dict", RESTRICTED_KEYS) +def test_openai_websocket_refusal_prefers_disabled_over_model_restriction(user_api_key_dict): + assert _openai_websocket_refusal(user_api_key_dict, MappingProxyType({})) is _OPENAI_WS_DISABLED_REFUSAL @pytest.mark.asyncio -@pytest.mark.parametrize( - "user_api_key_dict", - [ - UserAPIKeyAuth(), - UserAPIKeyAuth(models=["all-proxy-models"]), - UserAPIKeyAuth(models=["*"]), - UserAPIKeyAuth(models=["all-team-models"], team_models=["all-proxy-models"]), - ], -) +@pytest.mark.parametrize("user_api_key_dict", UNRESTRICTED_KEYS) async def test_openai_websocket_allows_unrestricted_keys(user_api_key_dict): - websocket = _mock_websocket("/openai/v1/responses", "") + websocket = _FakeWebSocket("/openai/v1/responses", "") - with ( - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router.get_credentials", - return_value="sk-provider", - ), - patch( - "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.websocket_passthrough_request", - new_callable=AsyncMock, - ) as mock_ws, - ): - await openai_websocket_proxy_route( - websocket=websocket, - endpoint="v1/responses", - user_api_key_dict=user_api_key_dict, - ) + with patch(GET_CREDENTIALS, return_value="sk-provider"): + relay = await _serve(websocket, "v1/responses", user_api_key_dict, ENABLED) - mock_ws.assert_awaited_once() - websocket.close.assert_not_awaited() + assert len(relay.calls) == 1 + assert websocket.sent == [] + assert websocket.closed is None diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index de1b7fe699e..cda1a3834ce 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25731,6 +25731,11 @@ export interface components { * @description If True and SSO is configured (MICROSOFT_CLIENT_ID, GOOGLE_CLIENT_ID, GENERIC_CLIENT_ID, or SAML_IDP_METADATA_URL/XML), disables username/password login on /login, /v2/login, and /v3/login so SSO is the only way to reach the Admin UI. An admin locked out of the UI can still administer the proxy over the API with the master key; unset this setting and restart the proxy to restore UI username/password login. Default is False. */ disable_password_login_when_sso_enabled?: boolean | null; + /** + * Enable Openai Websocket Passthrough + * @description Serve the OpenAI pass-through WebSocket route, which relays frames to OpenAI under the proxy's own provider credential without reading them. Off by default. + */ + enable_openai_websocket_passthrough?: boolean | null; /** * Enable Public Model Hub * @description Public model hub for users to see what models they have access to, supported openai params, etc. From 7351911b533717155599bdfcbf09701aa1760fe5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:39:56 -0700 Subject: [PATCH 2/9] fix(proxy): refuse OpenAI websocket passthrough on every enforced model allowlist and propagate the DB opt-in --- litellm/proxy/auth/auth_checks.py | 56 ++ .../llm_passthrough_endpoints.py | 37 +- litellm/proxy/proxy_server.py | 5 + .../proxy/auth/test_auth_checks.py | 602 +++++++----------- .../test_openai_ws_passthrough_routes.py | 128 ++-- tests/test_litellm/proxy/test_proxy_server.py | 82 ++- 6 files changed, 461 insertions(+), 449 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f83f0303deb..98d334ce2cc 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4155,6 +4155,62 @@ async def _granted_model_lists( ) +async def enforced_model_allowlists( + valid_token: UserAPIKeyAuth, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, +) -> tuple[Sequence[str], ...]: + """One model allowlist per level that ``common_checks`` enforces on a request from this identity.""" + key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token) + if prisma_client is None: + return (key_models,) + team_object: Final = ( + None + if valid_token.team_id is None + else await get_team_object( + team_id=valid_token.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + ) + user_object: Final = ( + None + if team_object is not None + else await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=proxy_logging_obj, + ) + ) + project_object: Final = ( + None + if valid_token.project_id is None + else await get_project_object( + project_id=valid_token.project_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + ) + return ( + key_models, + team_object.models if team_object is not None else (), + await _team_member_granted_models( + valid_token=valid_token, + team_object=team_object, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ), + user_object.models if user_object is not None else (), + project_object.models if project_object is not None else (), + ) + + async def collect_matched_model_access_groups( model: str | Sequence[str] | None, valid_token: UserAPIKeyAuth | None, diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 97d25e20939..e92c949299c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -13,7 +13,7 @@ import inspect import json import os import re -from collections.abc import AsyncGenerator, Callable, Mapping +from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol, cast @@ -36,6 +36,7 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse from litellm.proxy._types import * +from litellm.proxy.auth.auth_checks import enforced_model_allowlists from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( @@ -2341,9 +2342,8 @@ _OPENAI_WS_ALL_MODEL_ACCESS: Final = frozenset( ) -def _key_has_model_restrictions(user_api_key_dict: UserAPIKeyAuth) -> bool: - scoped_models: Final = (*user_api_key_dict.models, *user_api_key_dict.team_models) - return any(str(model) not in _OPENAI_WS_ALL_MODEL_ACCESS for model in scoped_models) +def _has_model_restrictions(model_allowlists: tuple[Sequence[str], ...]) -> bool: + return any(str(model) not in _OPENAI_WS_ALL_MODEL_ACCESS for allowlist in model_allowlists for model in allowlist) @dataclass(frozen=True, slots=True) @@ -2376,12 +2376,18 @@ def _is_openai_websocket_passthrough_enabled(general_settings: Mapping[str, obje return setting is True -def _openai_websocket_refusal( - user_api_key_dict: UserAPIKeyAuth, general_settings: Mapping[str, object] +class _OpenAIWebsocketModelAllowlists(Protocol): + async def __call__(self, valid_token: UserAPIKeyAuth, /) -> tuple[Sequence[str], ...]: ... + + +async def _openai_websocket_refusal( + user_api_key_dict: UserAPIKeyAuth, + general_settings: Mapping[str, object], + model_allowlists: _OpenAIWebsocketModelAllowlists, ) -> _OpenAIWebsocketRefusal | None: if not _is_openai_websocket_passthrough_enabled(general_settings): return _OPENAI_WS_DISABLED_REFUSAL - if _key_has_model_restrictions(user_api_key_dict): + if _has_model_restrictions(await model_allowlists(user_api_key_dict)): return _OPENAI_WS_MODEL_RESTRICTED_REFUSAL return None @@ -2410,6 +2416,20 @@ def _openai_websocket_relay() -> _OpenAIWebsocketRelay: return websocket_passthrough_request +def _proxy_model_allowlists() -> _OpenAIWebsocketModelAllowlists: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + async def resolve(valid_token: UserAPIKeyAuth, /) -> tuple[Sequence[str], ...]: + return await enforced_model_allowlists( + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + return resolve + + @router.websocket("/openai_passthrough/{endpoint:path}") @router.websocket("/openai/{endpoint:path}") async def openai_websocket_proxy_route( @@ -2418,6 +2438,7 @@ async def openai_websocket_proxy_route( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth_websocket)], general_settings: Annotated[Mapping[str, object], Depends(_proxy_general_settings)], relay: Annotated[_OpenAIWebsocketRelay, Depends(_openai_websocket_relay)], + model_allowlists: Annotated[_OpenAIWebsocketModelAllowlists, Depends(_proxy_model_allowlists)], ) -> None: """WebSocket passthrough for OpenAI prefixes (realtime / responses.connect).""" requested_subprotocols: Final = tuple( @@ -2427,7 +2448,7 @@ async def openai_websocket_proxy_route( ) negotiated_subprotocol: Final = requested_subprotocols[0] if requested_subprotocols else None - refusal: Final = _openai_websocket_refusal(user_api_key_dict, general_settings) + refusal: Final = await _openai_websocket_refusal(user_api_key_dict, general_settings, model_allowlists) if refusal is not None: await websocket.accept(subprotocol=negotiated_subprotocol) await websocket.send_text( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5a39b8c610a..e56395a6169 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6793,6 +6793,11 @@ class ProxyConfig: else: general_settings["apply_user_budget_to_team_keys"] = db_value if db_value is None else bool(db_value) + if "enable_openai_websocket_passthrough" not in self._yaml_general_settings_keys: + general_settings["enable_openai_websocket_passthrough"] = _general_settings.get( + "enable_openai_websocket_passthrough" + ) + ## STORE MODEL IN DB ## if "store_model_in_db" in _general_settings: value = _general_settings["store_model_in_db"] diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index be83ca57e76..45e10948267 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -126,14 +126,10 @@ def invalid_sso_user_defined_values(): def test_get_experimental_ui_login_jwt_auth_token_valid(valid_sso_user_defined_values): """Test generating JWT token with valid user role""" - token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( - valid_sso_user_defined_values - ) + token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(valid_sso_user_defined_values) # Decrypt and verify token contents - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") # Check that decrypted_token is not None before using json.loads assert decrypted_token is not None token_data = json.loads(decrypted_token) @@ -159,9 +155,7 @@ def test_get_cli_jwt_auth_token_includes_team_alias(valid_sso_user_defined_value team_alias="test-team", ) - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted_token is not None token_data = json.loads(decrypted_token) @@ -188,9 +182,7 @@ def test_get_cli_jwt_auth_token_carries_team_grants_not_user_allowlist( team_model_aliases={"team-fast": "gpt-4.1-mini"}, ) - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted_token is not None token_data = json.loads(decrypted_token) @@ -207,9 +199,7 @@ def test_get_cli_jwt_auth_token_keeps_user_allowlist_when_no_team( """A session token with no team bound still carries the user's own allowlist.""" token = ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values) - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted_token is not None token_data = json.loads(decrypted_token) @@ -222,12 +212,8 @@ def test_get_experimental_ui_login_jwt_auth_token_uses_10_min_expiry( valid_sso_user_defined_values, ): """Test that Experimental UI token uses fixed 10-minute expiry (does not use LITELLM_UI_SESSION_DURATION).""" - token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( - valid_sso_user_defined_values - ) - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(valid_sso_user_defined_values) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted_token is not None token_data = json.loads(decrypted_token) expires = datetime.fromisoformat(token_data["expires"].replace("Z", "+00:00")) @@ -244,43 +230,33 @@ def test_experimental_ui_token_ignores_litellm_ui_session_duration( Experimental UI intentionally uses fixed 10-min expiry. If this test fails, the constant was incorrectly wired to the experimental flow.""" # Default LITELLM_UI_SESSION_DURATION is "24h" - token must still expire in ~10 min - token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( - valid_sso_user_defined_values - ) - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(valid_sso_user_defined_values) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted_token is not None token_data = json.loads(decrypted_token) expires = datetime.fromisoformat(token_data["expires"].replace("Z", "+00:00")) now = get_utc_datetime() # Must be ~10 min, NOT 24h. If LITELLM_UI_SESSION_DURATION were incorrectly used, this would fail. - assert expires <= now + timedelta( - minutes=11 - ), "Experimental UI must use 10-min expiry, not LITELLM_UI_SESSION_DURATION" + assert expires <= now + timedelta(minutes=11), ( + "Experimental UI must use 10-min expiry, not LITELLM_UI_SESSION_DURATION" + ) def test_get_experimental_ui_login_jwt_auth_token_invalid( invalid_sso_user_defined_values, ): """Test generating JWT token with missing user role""" - with pytest.raises(Exception, match='User role is required for experimental UI login') as exc_info: - ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( - invalid_sso_user_defined_values - ) + with pytest.raises(Exception, match="User role is required for experimental UI login") as exc_info: + ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(invalid_sso_user_defined_values) assert str(exc_info.value) == "User role is required for experimental UI login" -def test_get_key_object_from_ui_hash_key_valid( - valid_sso_user_defined_values, monkeypatch -): +def test_get_key_object_from_ui_hash_key_valid(valid_sso_user_defined_values, monkeypatch): """Test getting key object from valid UI hash key""" monkeypatch.setenv("EXPERIMENTAL_UI_LOGIN", "True") # Generate a valid token - token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token( - valid_sso_user_defined_values - ) + token = ExperimentalUIJWTToken.get_experimental_ui_login_jwt_auth_token(valid_sso_user_defined_values) # Get key object key_object = ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) @@ -309,9 +285,7 @@ def test_get_key_object_from_ui_hash_key_invalid(): ("project", ProxyErrorTypes.project_model_access_denied), ], ) -def test_can_object_call_model_denials_return_forbidden( - object_type, expected_error_type -): +def test_can_object_call_model_denials_return_forbidden(object_type, expected_error_type): with pytest.raises(ProxyException) as exc_info: _can_object_call_model( model="restricted-model", @@ -568,9 +542,7 @@ async def test_get_key_object_should_reconnect_once_on_db_connection_error(): @pytest.mark.asyncio async def test_get_key_object_should_raise_if_reconnect_fails_on_db_connection_error(): mock_prisma_client = MagicMock() - mock_prisma_client.get_data = AsyncMock( - side_effect=httpx.ConnectError("db not reachable after outage") - ) + mock_prisma_client.get_data = AsyncMock(side_effect=httpx.ConnectError("db not reachable after outage")) mock_prisma_client.attempt_db_reconnect = AsyncMock(return_value=False) mock_cache = MagicMock() @@ -613,9 +585,7 @@ class TestAuthCacheRedisWritePolicy: @pytest.mark.asyncio async def test_get_key_object_db_load_publishes_to_redis(self): mock_prisma_client = MagicMock() - mock_prisma_client.get_data = AsyncMock( - return_value=UserAPIKeyAuth(token="hashed-token-db") - ) + mock_prisma_client.get_data = AsyncMock(return_value=UserAPIKeyAuth(token="hashed-token-db")) fake_redis = _fake_redis_cache() cache = UserApiKeyCache() @@ -630,8 +600,7 @@ class TestAuthCacheRedisWritePolicy: assert key_obj.token == "hashed-token-db" fake_redis.async_set_cache.assert_awaited_once() assert ( - fake_redis.async_set_cache.await_args.kwargs.get("key") - or fake_redis.async_set_cache.await_args.args[0] + fake_redis.async_set_cache.await_args.kwargs.get("key") or fake_redis.async_set_cache.await_args.args[0] ) == "hashed-token-db" @@ -640,9 +609,7 @@ def test_get_cli_jwt_auth_token_default_expiration(valid_sso_user_defined_values token = ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values) # Decrypt and verify token contents - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted_token is not None token_data = json.loads(decrypted_token) @@ -664,9 +631,7 @@ def test_get_cli_jwt_auth_token_default_expiration(valid_sso_user_defined_values assert expires >= get_utc_datetime() + timedelta(hours=23, minutes=59) -def test_get_cli_jwt_auth_token_custom_expiration( - valid_sso_user_defined_values, monkeypatch -): +def test_get_cli_jwt_auth_token_custom_expiration(valid_sso_user_defined_values, monkeypatch): """Test generating CLI JWT token with custom expiration via environment variable""" import importlib @@ -681,14 +646,10 @@ def test_get_cli_jwt_auth_token_custom_expiration( # Also reload auth_checks to pick up the new constant value importlib.reload(auth_checks) - token = auth_checks.ExperimentalUIJWTToken.get_cli_jwt_auth_token( - valid_sso_user_defined_values - ) + token = auth_checks.ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values) # Decrypt and verify token contents - decrypted_token = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + decrypted_token = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted_token is not None token_data = json.loads(decrypted_token) @@ -706,18 +667,12 @@ def test_get_cli_jwt_auth_token_unique_per_session(valid_sso_user_defined_values from litellm.constants import CLI_SESSION_KEY_PREFIX def _decode(token: str) -> dict: - decrypted = decrypt_value_helper( - token, key="ui_hash_key", exception_type="debug" - ) + decrypted = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted is not None return json.loads(decrypted) - first = _decode( - ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values) - ) - second = _decode( - ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values) - ) + first = _decode(ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values)) + second = _decode(ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values)) assert first["token"].startswith(f"{CLI_SESSION_KEY_PREFIX}-") assert second["token"].startswith(f"{CLI_SESSION_KEY_PREFIX}-") @@ -740,9 +695,7 @@ def test_get_cli_jwt_auth_token_applies_fallback_budget(valid_sso_user_defined_v def test_get_cli_jwt_auth_token_no_fallback_when_budget_provided( valid_sso_user_defined_values, ): - token = ExperimentalUIJWTToken.get_cli_jwt_auth_token( - valid_sso_user_defined_values, max_budget=None - ) + token = ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values, max_budget=None) decrypted = decrypt_value_helper(token, key="ui_hash_key", exception_type="debug") assert decrypted is not None assert json.loads(decrypted).get("max_budget") is None @@ -945,9 +898,7 @@ async def test_get_user_object_upsert_includes_user_email(): mock_prisma_client.db.litellm_usertable.create.assert_called_once() creation_args = mock_prisma_client.db.litellm_usertable.create.call_args[1]["data"] - assert ( - "user_email" in creation_args - ), "user_email should be included when upserting a new user" + assert "user_email" in creation_args, "user_email should be included when upserting a new user" assert creation_args["user_email"] == "test@example.com" assert creation_args["user_id"] == "new_test_user" @@ -962,12 +913,8 @@ async def test_get_user_object_backfills_null_email_from_cache_hit(): was returned unchanged and the DB was never updated. """ cache = UserApiKeyCache() - existing = LiteLLM_UserTable( - user_id="jwt-user-1", user_email=None, user_role="internal_user" - ) - await cache.async_set_cache( - key="jwt-user-1", value=existing, model_type=LiteLLM_UserTable - ) + existing = LiteLLM_UserTable(user_id="jwt-user-1", user_email=None, user_role="internal_user") + await cache.async_set_cache(key="jwt-user-1", value=existing, model_type=LiteLLM_UserTable) mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=1) @@ -996,9 +943,7 @@ async def test_get_user_object_backfills_null_email_from_cache_hit(): assert update_kwargs["where"] == {"user_id": "jwt-user-1", "user_email": None} assert update_kwargs["data"]["user_email"] == "jwt-user-1@example.com" - refreshed = await cache.async_get_cache( - key="jwt-user-1", model_type=LiteLLM_UserTable - ) + refreshed = await cache.async_get_cache(key="jwt-user-1", model_type=LiteLLM_UserTable) assert refreshed is not None assert refreshed.user_email == "jwt-user-1@example.com" @@ -1010,9 +955,7 @@ async def test_get_user_object_backfills_null_email_from_db_read(): backfilled from the JWT-provided email before it is cached and returned. """ cache = UserApiKeyCache() - db_row = LiteLLM_UserTable( - user_id="jwt-user-3", user_email=None, user_role="internal_user" - ) + db_row = LiteLLM_UserTable(user_id="jwt-user-3", user_email=None, user_role="internal_user") backfilled_row = LiteLLM_UserTable( user_id="jwt-user-3", user_email="jwt-user-3@example.com", @@ -1020,15 +963,11 @@ async def test_get_user_object_backfills_null_email_from_db_read(): ) mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( - side_effect=[db_row, backfilled_row] - ) + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(side_effect=[db_row, backfilled_row]) mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=None) mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=1) - with patch( - "litellm.proxy.auth.auth_checks._should_check_db", return_value=True - ): + with patch("litellm.proxy.auth.auth_checks._should_check_db", return_value=True): result = await get_user_object( user_id="jwt-user-3", prisma_client=mock_prisma_client, @@ -1042,9 +981,7 @@ async def test_get_user_object_backfills_null_email_from_db_read(): assert result.user_email == "jwt-user-3@example.com" mock_prisma_client.db.litellm_usertable.update_many.assert_called_once() - refreshed = await cache.async_get_cache( - key="jwt-user-3", model_type=LiteLLM_UserTable - ) + refreshed = await cache.async_get_cache(key="jwt-user-3", model_type=LiteLLM_UserTable) assert refreshed is not None assert refreshed.user_email == "jwt-user-3@example.com" @@ -1062,9 +999,7 @@ async def test_get_user_object_does_not_overwrite_existing_email(): user_email="operator-set@example.com", user_role="internal_user", ) - await cache.async_set_cache( - key="jwt-user-2", value=existing, model_type=LiteLLM_UserTable - ) + await cache.async_set_cache(key="jwt-user-2", value=existing, model_type=LiteLLM_UserTable) mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=0) @@ -1091,12 +1026,8 @@ async def test_get_user_object_backfill_race_prefers_db_email(): with the value the DB accepted, not this request's proposed email. """ cache = UserApiKeyCache() - existing = LiteLLM_UserTable( - user_id="jwt-user-4", user_email=None, user_role="internal_user" - ) - await cache.async_set_cache( - key="jwt-user-4", value=existing, model_type=LiteLLM_UserTable - ) + existing = LiteLLM_UserTable(user_id="jwt-user-4", user_email=None, user_role="internal_user") + await cache.async_set_cache(key="jwt-user-4", value=existing, model_type=LiteLLM_UserTable) winner_row = LiteLLM_UserTable( user_id="jwt-user-4", @@ -1105,9 +1036,7 @@ async def test_get_user_object_backfill_race_prefers_db_email(): ) mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=0) - mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( - return_value=winner_row - ) + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=winner_row) result = await get_user_object( user_id="jwt-user-4", @@ -1121,9 +1050,7 @@ async def test_get_user_object_backfill_race_prefers_db_email(): assert result is not None assert result.user_email == "winner@example.com" - refreshed = await cache.async_get_cache( - key="jwt-user-4", model_type=LiteLLM_UserTable - ) + refreshed = await cache.async_get_cache(key="jwt-user-4", model_type=LiteLLM_UserTable) assert refreshed is not None assert refreshed.user_email == "winner@example.com" @@ -1138,12 +1065,8 @@ async def test_get_user_object_backfill_caches_persisted_email_not_proposed(): optimistically caching the proposed email would serve a stale value. """ cache = UserApiKeyCache() - existing = LiteLLM_UserTable( - user_id="jwt-user-5", user_email=None, user_role="internal_user" - ) - await cache.async_set_cache( - key="jwt-user-5", value=existing, model_type=LiteLLM_UserTable - ) + existing = LiteLLM_UserTable(user_id="jwt-user-5", user_email=None, user_role="internal_user") + await cache.async_set_cache(key="jwt-user-5", value=existing, model_type=LiteLLM_UserTable) persisted_row = LiteLLM_UserTable( user_id="jwt-user-5", @@ -1152,9 +1075,7 @@ async def test_get_user_object_backfill_caches_persisted_email_not_proposed(): ) mock_prisma_client = MagicMock() mock_prisma_client.db.litellm_usertable.update_many = AsyncMock(return_value=1) - mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock( - return_value=persisted_row - ) + mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=persisted_row) result = await get_user_object( user_id="jwt-user-5", @@ -1168,9 +1089,7 @@ async def test_get_user_object_backfill_caches_persisted_email_not_proposed(): assert result is not None assert result.user_email == "admin-edited@example.com" - refreshed = await cache.async_get_cache( - key="jwt-user-5", model_type=LiteLLM_UserTable - ) + refreshed = await cache.async_get_cache(key="jwt-user-5", model_type=LiteLLM_UserTable) assert refreshed is not None assert refreshed.user_email == "admin-edited@example.com" @@ -1224,10 +1143,7 @@ async def test_get_user_object_upsert_routes_default_team_to_membership(monkeypa mock_add_to_team.assert_awaited_once() passed_teams = mock_add_to_team.await_args[1]["teams"] assert [team.team_id for team in passed_teams] == ["default-team"] - assert ( - mock_add_to_team.await_args[1]["user_api_key_dict"].user_role - == LitellmUserRoles.PROXY_ADMIN - ) + assert mock_add_to_team.await_args[1]["user_api_key_dict"].user_role == LitellmUserRoles.PROXY_ADMIN def test_log_budget_lookup_failure_dry_run(): @@ -1252,9 +1168,7 @@ def test_log_budget_lookup_failure_skips_user_not_found(): @pytest.mark.asyncio -@patch( - "litellm.proxy.management_endpoints.team_endpoints.new_team", new_callable=AsyncMock -) +@patch("litellm.proxy.management_endpoints.team_endpoints.new_team", new_callable=AsyncMock) async def test_get_team_db_check_calls_new_team_on_upsert(mock_new_team, monkeypatch): """ Test that _get_team_db_check correctly calls the `new_team` function @@ -1288,12 +1202,8 @@ async def test_get_team_db_check_calls_new_team_on_upsert(mock_new_team, monkeyp @pytest.mark.asyncio -@patch( - "litellm.proxy.management_endpoints.team_endpoints.new_team", new_callable=AsyncMock -) -async def test_get_team_db_check_does_not_call_new_team_if_exists( - mock_new_team, monkeypatch -): +@patch("litellm.proxy.management_endpoints.team_endpoints.new_team", new_callable=AsyncMock) +async def test_get_team_db_check_does_not_call_new_team_if_exists(mock_new_team, monkeypatch): """ Test that _get_team_db_check does NOT call the `new_team` function if the team already exists in the database. @@ -1327,9 +1237,7 @@ async def test_get_team_db_check_does_not_call_new_team_if_exists( (MagicMock(), MagicMock(), True), # No vector stores to run ], ) -async def test_vector_store_access_check_early_returns( - prisma_client, vector_store_registry, expected_result -): +async def test_vector_store_access_check_early_returns(prisma_client, vector_store_registry, expected_result): """Test vector_store_access_check returns True for early exit conditions""" request_body = {"messages": [{"role": "user", "content": "test"}]} @@ -1411,9 +1319,7 @@ async def test_vector_store_access_check_skips_db_lookup_when_no_vector_stores_r ), # Partial access ], ) -def test_can_object_call_vector_stores_scenarios( - object_permissions, vector_store_ids, should_raise, error_type -): +def test_can_object_call_vector_stores_scenarios(object_permissions, vector_store_ids, should_raise, error_type): """Test _can_object_call_vector_stores with various permission scenarios""" # Convert dict to object if not None if object_permissions is not None: @@ -1421,11 +1327,7 @@ def test_can_object_call_vector_stores_scenarios( mock_permissions.vector_stores = object_permissions["vector_stores"] object_permissions = mock_permissions - object_type = ( - "key" - if error_type == ProxyErrorTypes.key_vector_store_access_denied - else "team" - ) + object_type = "key" if error_type == ProxyErrorTypes.key_vector_store_access_denied else "team" if should_raise: with pytest.raises(ProxyException) as exc_info: @@ -1460,9 +1362,7 @@ async def test_vector_store_access_check_with_permissions(): mock_prisma_client = MagicMock() mock_permissions = MagicMock() mock_permissions.vector_stores = ["store-1", "store-2"] - mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock( - return_value=mock_permissions - ) + mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=mock_permissions) mock_vector_store_registry = MagicMock() mock_vector_store_registry.get_vector_store_ids_to_run.return_value = ["store-1"] @@ -1508,14 +1408,10 @@ async def test_vector_store_access_check_with_team_permissions(): mock_prisma_client = MagicMock() team_permissions = MagicMock() team_permissions.vector_stores = ["team-store-allowed"] - mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock( - return_value=team_permissions - ) + mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions) mock_vector_store_registry = MagicMock() - mock_vector_store_registry.get_vector_store_ids_to_run.return_value = [ - "team-store-allowed" - ] + mock_vector_store_registry.get_vector_store_ids_to_run.return_value = ["team-store-allowed"] with ( patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), @@ -1529,9 +1425,7 @@ async def test_vector_store_access_check_with_team_permissions(): assert result is True - mock_vector_store_registry.get_vector_store_ids_to_run.return_value = [ - "team-store-denied" - ] + mock_vector_store_registry.get_vector_store_ids_to_run.return_value = ["team-store-denied"] with ( patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client), @@ -2092,9 +1986,7 @@ async def test_get_tag_objects_batch(): mock_cache.async_set_cache = AsyncMock() # Mock DB to return all uncached tags in ONE query - mock_prisma.db.litellm_tagtable.find_many = AsyncMock( - return_value=[uncached_tag_1, uncached_tag_2, uncached_tag_3] - ) + mock_prisma.db.litellm_tagtable.find_many = AsyncMock(return_value=[uncached_tag_1, uncached_tag_2, uncached_tag_3]) # Call batch fetch tag_objects = await get_tag_objects_batch( @@ -2196,9 +2088,7 @@ async def test_get_tag_objects_batch_never_queries_db_for_unregistered_tags(): from litellm.proxy.auth.auth_checks import get_tag_objects_batch mock_prisma = MagicMock() - mock_prisma.db.litellm_tagtable.find_many = AsyncMock( - return_value=[_tag_registry_row("some-other-tag")] - ) + mock_prisma.db.litellm_tagtable.find_many = AsyncMock(return_value=[_tag_registry_row("some-other-tag")]) cache = UserApiKeyCache() first = await get_tag_objects_batch( @@ -2209,9 +2099,7 @@ async def test_get_tag_objects_batch_never_queries_db_for_unregistered_tags(): assert first == {} # The only query is the names-only registry fetch; the tag itself is never looked up. - mock_prisma.db.litellm_tagtable.find_many.assert_called_once_with( - take=TAG_REGISTRY_MAX_SIZE + 1 - ) + mock_prisma.db.litellm_tagtable.find_many.assert_called_once_with(take=TAG_REGISTRY_MAX_SIZE + 1) second = await get_tag_objects_batch( tag_names=["unregistered-tag"], @@ -2380,9 +2268,7 @@ async def test_get_tag_objects_batch_oversized_registry_falls_back_and_stops_ref """Past the cap the registry is unusable: keep the old per-tag path, but stop rebuilding it.""" from litellm.proxy.auth.auth_checks import get_tag_objects_batch - oversized = [ - _tag_registry_row(f"tag-{index}") for index in range(TAG_REGISTRY_MAX_SIZE + 1) - ] + oversized = [_tag_registry_row(f"tag-{index}") for index in range(TAG_REGISTRY_MAX_SIZE + 1)] async def fake_find_many(**kwargs): if "where" not in kwargs: @@ -2399,10 +2285,7 @@ async def test_get_tag_objects_batch_oversized_registry_falls_back_and_stops_ref user_api_key_cache=cache, ) assert list(first) == ["tag-a"] - assert ( - await cache.async_get_cache(key=tag_registry_cache_key()) - == TAG_REGISTRY_OVERFLOW_SENTINEL - ) + assert await cache.async_get_cache(key=tag_registry_cache_key()) == TAG_REGISTRY_OVERFLOW_SENTINEL second = await get_tag_objects_batch( tag_names=["tag-b"], @@ -2427,17 +2310,12 @@ async def test_tag_max_budget_check_still_enforces_registered_tag_over_budget(): async def fake_find_many(**kwargs): if "where" not in kwargs: return [_tag_registry_row("paid-tag")] - return [ - _tag_db_row(name, max_budget=1.0) - for name in kwargs["where"]["tag_name"]["in"] - ] + return [_tag_db_row(name, max_budget=1.0) for name in kwargs["where"]["tag_name"]["in"]] mock_prisma = MagicMock() mock_prisma.db.litellm_tagtable.find_many = AsyncMock(side_effect=fake_find_many) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:tag:paid-tag": return 1.5 return fallback_spend @@ -2846,8 +2724,7 @@ def _pass_through_request() -> Request: LITELLM_PASS_THROUGH_ENDPOINT_MARKER, ) - def pass_through_endpoint(): - ... + def pass_through_endpoint(): ... setattr(pass_through_endpoint, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, True) return Request(scope={"type": "http", "headers": [], "endpoint": pass_through_endpoint}) @@ -2857,8 +2734,7 @@ def _builtin_request() -> Request: """A Request dispatched to a built-in (non-pass-through) handler, e.g. what a custom path colliding with a core route actually resolves to.""" - def chat_completions(): - ... + def chat_completions(): ... return Request(scope={"type": "http", "headers": [], "endpoint": chat_completions}) @@ -3023,9 +2899,7 @@ async def test_virtual_key_soft_budget_check_without_user_obj(): ], ) @pytest.mark.asyncio -async def test_virtual_key_soft_budget_check_scenarios( - spend, soft_budget, expect_alert -): +async def test_virtual_key_soft_budget_check_scenarios(spend, soft_budget, expect_alert): """Test _virtual_key_soft_budget_check with various spend and soft_budget scenarios""" alert_triggered = False @@ -3054,9 +2928,9 @@ async def test_virtual_key_soft_budget_check_scenarios( await asyncio.sleep(0.1) - assert ( - alert_triggered == expect_alert - ), f"Expected alert_triggered to be {expect_alert} for spend={spend}, soft_budget={soft_budget}" + assert alert_triggered == expect_alert, ( + f"Expected alert_triggered to be {expect_alert} for spend={spend}, soft_budget={soft_budget}" + ) @pytest.mark.asyncio @@ -3167,9 +3041,7 @@ async def test_virtual_key_max_budget_alert_check_without_user_obj(): ], ) @pytest.mark.asyncio -async def test_virtual_key_max_budget_alert_check_scenarios( - spend, max_budget, expect_alert -): +async def test_virtual_key_max_budget_alert_check_scenarios(spend, max_budget, expect_alert): """Test _virtual_key_max_budget_alert_check with various spend and max_budget scenarios""" alert_triggered = False @@ -3198,9 +3070,9 @@ async def test_virtual_key_max_budget_alert_check_scenarios( await asyncio.sleep(0.1) - assert ( - alert_triggered == expect_alert - ), f"Expected alert_triggered to be {expect_alert} for spend={spend}, max_budget={max_budget}" + assert alert_triggered == expect_alert, ( + f"Expected alert_triggered to be {expect_alert} for spend={spend}, max_budget={max_budget}" + ) @pytest.mark.asyncio @@ -3459,9 +3331,7 @@ async def test_custom_auth_common_checks_opt_in(): "prisma_client": None, "user_api_key_cache": MagicMock(), "proxy_logging_obj": MagicMock(), - "general_settings": ( - {"custom_auth_run_common_checks": True} if flag else {} - ), + "general_settings": ({"custom_auth_run_common_checks": True} if flag else {}), "llm_router": None, "user_custom_auth": user_custom_auth, "litellm_proxy_admin_name": "admin", @@ -3533,9 +3403,7 @@ async def test_virtual_key_budget_check_reads_from_spend_counter(): proxy_logging_obj = ProxyLogging(user_api_key_cache=None) proxy_logging_obj.budget_alerts = AsyncMock() - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:key:test-hashed-token": return 1.5 return fallback_spend @@ -3569,9 +3437,7 @@ async def test_virtual_key_budget_check_fallback_no_counter(): proxy_logging_obj.budget_alerts = AsyncMock() # get_current_spend returns fallback_spend when no counter exists - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): return fallback_spend with patch("litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend): @@ -3601,9 +3467,7 @@ def _over_budget_token(**overrides) -> UserAPIKeyAuth: def _patched_spend(value: float): - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): return value return patch("litellm.proxy.proxy_server.get_current_spend", mock_get_current_spend) @@ -3664,9 +3528,7 @@ async def test_budget_throttle_decision_cleared_before_caching(): otherwise it would re-apply (and compound) on every subsequent request.""" from litellm.proxy.auth.auth_checks import _copy_user_api_key_auth_for_cache - valid_token = _over_budget_token( - tpm_limit=1000, rpm_limit=100, metadata={"throttle_on_budget_exceeded": True} - ) + valid_token = _over_budget_token(tpm_limit=1000, rpm_limit=100, metadata={"throttle_on_budget_exceeded": True}) valid_token.budget_throttle_pct = 0.1 cached = _copy_user_api_key_auth_for_cache(user_api_key_obj=valid_token) @@ -3762,9 +3624,7 @@ async def test_team_budget_check_reads_from_spend_counter(): proxy_logging_obj = ProxyLogging(user_api_key_cache=None) proxy_logging_obj.budget_alerts = AsyncMock() - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team:test-team": return 1.5 return fallback_spend @@ -3791,9 +3651,7 @@ async def test_end_user_budget_check_reads_from_spend_counter(): litellm_budget_table=LiteLLM_BudgetTable(max_budget=1.0), ) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:end_user:customer-1": return 1.5 return fallback_spend @@ -3821,9 +3679,7 @@ async def test_tag_budget_check_reads_from_spend_counter(): litellm_budget_table=LiteLLM_BudgetTable(max_budget=1.0), ) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:tag:paid-tag": return 1.5 return fallback_spend @@ -3873,9 +3729,7 @@ async def test_team_member_budget_check_reads_from_spend_counter(): proxy_logging_obj = ProxyLogging(user_api_key_cache=None) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team_member:test-user:test-team": return 1.5 return fallback_spend @@ -3915,9 +3769,7 @@ class TestGuardrailModificationCheck: team_object = MagicMock() team_object.metadata = {} # no permission - return _guardrail_modification_check( - request_body=request_body, team_object=team_object - ) + return _guardrail_modification_check(request_body=request_body, team_object=team_object) def test_noop_when_no_guardrail_keys_present(self): # no-op — should return silently @@ -3965,9 +3817,7 @@ class TestGuardrailModificationCheck: return_value=False, ): with pytest.raises(HTTPException) as exc: - self._call( - {"metadata": {"opted_out_global_guardrails": ["some_guardrail"]}} - ) + self._call({"metadata": {"opted_out_global_guardrails": ["some_guardrail"]}}) assert exc.value.status_code == 403 @pytest.mark.parametrize( @@ -4101,18 +3951,12 @@ async def test_team_member_budget_check_falls_back_to_team_default_budget_id(): fake_budget_row = MagicMock() fake_budget_row.max_budget = 50.0 - fake_budget_row.dict = MagicMock( - return_value={"budget_id": "budget-default", "max_budget": 50.0} - ) + fake_budget_row.dict = MagicMock(return_value={"budget_id": "budget-default", "max_budget": 50.0}) prisma_client = MagicMock() - prisma_client.db.litellm_budgettable.find_unique = AsyncMock( - return_value=fake_budget_row - ) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock(return_value=fake_budget_row) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team_member:test-user:test-team": return 70.0 return fallback_spend @@ -4203,15 +4047,11 @@ async def test_team_member_budget_check_per_member_override_wins_over_team_defau fake_budget_row.max_budget = 50.0 prisma_client = MagicMock() - prisma_client.db.litellm_budgettable.find_unique = AsyncMock( - return_value=fake_budget_row - ) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock(return_value=fake_budget_row) mocked_spend = 70.0 - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team_member:test-user:test-team": return mocked_spend return fallback_spend @@ -4292,18 +4132,12 @@ async def test_team_member_budget_check_null_clone_falls_back_to_team_default(): fake_default_row = MagicMock() fake_default_row.max_budget = 65.0 - fake_default_row.dict = MagicMock( - return_value={"budget_id": "budget-default", "max_budget": 65.0} - ) + fake_default_row.dict = MagicMock(return_value={"budget_id": "budget-default", "max_budget": 65.0}) prisma_client = MagicMock() - prisma_client.db.litellm_budgettable.find_unique = AsyncMock( - return_value=fake_default_row - ) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock(return_value=fake_default_row) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team_member:test-user:test-team": return 500.0 return fallback_spend @@ -4361,18 +4195,12 @@ async def test_team_member_budget_check_null_clone_with_null_default_skips_enfor fake_default_row = MagicMock() fake_default_row.max_budget = None - fake_default_row.dict = MagicMock( - return_value={"budget_id": "budget-default", "max_budget": None} - ) + fake_default_row.dict = MagicMock(return_value={"budget_id": "budget-default", "max_budget": None}) prisma_client = MagicMock() - prisma_client.db.litellm_budgettable.find_unique = AsyncMock( - return_value=fake_default_row - ) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock(return_value=fake_default_row) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team_member:test-user:test-team": return 1000.0 return fallback_spend @@ -4430,18 +4258,12 @@ async def test_team_member_budget_check_zero_team_default_treated_as_no_cap(): # Team default budget row with max_budget=0.0 (the regression trigger). fake_default_row = MagicMock() fake_default_row.max_budget = 0.0 - fake_default_row.dict = MagicMock( - return_value={"budget_id": "budget-default", "max_budget": 0.0} - ) + fake_default_row.dict = MagicMock(return_value={"budget_id": "budget-default", "max_budget": 0.0}) prisma_client = MagicMock() - prisma_client.db.litellm_budgettable.find_unique = AsyncMock( - return_value=fake_default_row - ) + prisma_client.db.litellm_budgettable.find_unique = AsyncMock(return_value=fake_default_row) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team_member:test-user:test-team": return 0.0 return fallback_spend @@ -4499,9 +4321,7 @@ async def test_team_member_budget_check_zero_per_member_row_still_blocks(): prisma_client = MagicMock() prisma_client.db.litellm_budgettable.find_unique = AsyncMock(return_value=None) - async def mock_get_current_spend( - counter_key, fallback_spend, max_budget=None, **kwargs - ): + async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs): if counter_key == "spend:team_member:test-user:test-team": return 0.0 return fallback_spend @@ -4549,19 +4369,13 @@ def _patch_validation_helpers(monkeypatch, *, end_user=None, user=None, fuzzy=No """Stub out the DB helpers resolve_and_validate_end_user_id delegates to.""" from litellm.proxy.auth import auth_checks - monkeypatch.setattr( - auth_checks, "get_end_user_object", AsyncMock(return_value=end_user) - ) + monkeypatch.setattr(auth_checks, "get_end_user_object", AsyncMock(return_value=end_user)) monkeypatch.setattr(auth_checks, "get_user_object", AsyncMock(return_value=user)) - monkeypatch.setattr( - auth_checks, "_get_fuzzy_user_object", AsyncMock(return_value=fuzzy) - ) + monkeypatch.setattr(auth_checks, "_get_fuzzy_user_object", AsyncMock(return_value=fuzzy)) @pytest.mark.asyncio -async def test_resolve_end_user_returns_none_for_none_input( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_returns_none_for_none_input(_validate_flag_on, monkeypatch): from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id _patch_validation_helpers(monkeypatch) @@ -4596,9 +4410,7 @@ async def test_resolve_end_user_passes_through_when_flag_disabled(monkeypatch): @pytest.mark.asyncio -async def test_resolve_end_user_passes_through_when_no_prisma_client( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_passes_through_when_no_prisma_client(_validate_flag_on, monkeypatch): from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id _patch_validation_helpers(monkeypatch) @@ -4632,9 +4444,7 @@ async def test_resolve_end_user_matches_end_user_table(_validate_flag_on, monkey @pytest.mark.asyncio -async def test_resolve_end_user_matches_user_table_by_user_id( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_matches_user_table_by_user_id(_validate_flag_on, monkeypatch): from litellm.proxy.auth import auth_checks from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id @@ -4652,9 +4462,7 @@ async def test_resolve_end_user_matches_user_table_by_user_id( @pytest.mark.asyncio -async def test_resolve_end_user_matches_user_table_by_email( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_matches_user_table_by_email(_validate_flag_on, monkeypatch): """Email-shaped ids route through get_user_object with user_email set. The fuzzy lookup must happen inside get_user_object so it shares the @@ -4682,9 +4490,7 @@ async def test_resolve_end_user_matches_user_table_by_email( @pytest.mark.asyncio -async def test_resolve_end_user_non_email_id_does_not_pass_user_email( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_non_email_id_does_not_pass_user_email(_validate_flag_on, monkeypatch): """Non-email ids skip the email fuzzy path to avoid a pointless DB hit.""" from litellm.proxy.auth import auth_checks from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id @@ -4703,9 +4509,7 @@ async def test_resolve_end_user_non_email_id_does_not_pass_user_email( @pytest.mark.asyncio -async def test_resolve_end_user_drops_codex_opaque_identifier( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_drops_codex_opaque_identifier(_validate_flag_on, monkeypatch): from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id _patch_validation_helpers(monkeypatch) # all helpers return None @@ -4727,9 +4531,7 @@ async def test_resolve_end_user_drops_codex_opaque_identifier( @pytest.mark.asyncio -async def test_resolve_end_user_preserves_id_when_default_budget_configured( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_preserves_id_when_default_budget_configured(_validate_flag_on, monkeypatch): """Don't drop unregistered ids when litellm.max_end_user_budget_id is set. The default end-user budget is applied downstream when the id is present @@ -4766,9 +4568,7 @@ async def test_resolve_end_user_drops_unknown_email(_validate_flag_on, monkeypat @pytest.mark.asyncio -async def test_resolve_end_user_uses_cached_valid_result( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_uses_cached_valid_result(_validate_flag_on, monkeypatch): from litellm.proxy.auth import auth_checks from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id @@ -4788,9 +4588,7 @@ async def test_resolve_end_user_uses_cached_valid_result( @pytest.mark.asyncio -async def test_resolve_end_user_uses_cached_invalid_result( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_uses_cached_invalid_result(_validate_flag_on, monkeypatch): from litellm.proxy.auth import auth_checks from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id @@ -4809,9 +4607,7 @@ async def test_resolve_end_user_uses_cached_invalid_result( @pytest.mark.asyncio -async def test_resolve_end_user_swallows_db_errors_and_returns_none( - _validate_flag_on, monkeypatch -): +async def test_resolve_end_user_swallows_db_errors_and_returns_none(_validate_flag_on, monkeypatch): from litellm.proxy.auth import auth_checks from litellm.proxy.auth.auth_checks import resolve_and_validate_end_user_id @@ -4932,19 +4728,13 @@ async def test_cache_team_object_writes_team_id_and_invalidates_team_alias(): ) # (1) team_id-keyed write fires with the refreshed object - written_keys = [ - (c.kwargs.get("key") or c.args[0]) - for c in cache.async_set_cache.await_args_list - ] + written_keys = [(c.kwargs.get("key") or c.args[0]) for c in cache.async_set_cache.await_args_list] assert written_keys == ["team_id:team-1234"], ( "Only the team_id-keyed write should fire; the alias key must be " "deleted, NOT written. " f"Got writes: {written_keys}" ) - written_value = ( - cache.async_set_cache.await_args.kwargs.get("value") - or cache.async_set_cache.await_args.args[1] - ) + written_value = cache.async_set_cache.await_args.kwargs.get("value") or cache.async_set_cache.await_args.args[1] assert written_value is team_table # (2) team_alias-keyed entry is deleted in BOTH the in-memory cache @@ -4978,10 +4768,7 @@ async def test_cache_team_object_writes_team_id_and_invalidates_team_alias(): logging_obj2.internal_usage_cache.dual_cache.async_delete_cache.assert_awaited_once_with( key="team_id:team-no-alias" ) - written_keys_aliasless = [ - (c.kwargs.get("key") or c.args[0]) - for c in cache2.async_set_cache.await_args_list - ] + written_keys_aliasless = [(c.kwargs.get("key") or c.args[0]) for c in cache2.async_set_cache.await_args_list] assert written_keys_aliasless == ["team_id:team-no-alias"] @@ -5061,9 +4848,7 @@ async def test_team_update_not_shadowed_by_internal_usage_cache_lit_4391(): await _cache_team_object( team_id=team_id, - team_table=LiteLLM_TeamTableCachedObj( - team_id=team_id, models=["model-a", "model-b"] - ), + team_table=LiteLLM_TeamTableCachedObj(team_id=team_id, models=["model-a", "model-b"]), user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) @@ -5173,9 +4958,7 @@ async def test_cache_team_object_tolerates_cache_invalidation_failures(): cache.async_set_cache = AsyncMock() cache.delete_cache = MagicMock(side_effect=Exception("redis down")) logging_obj = MagicMock() - logging_obj.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock( - side_effect=Exception("redis down") - ) + logging_obj.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock(side_effect=Exception("redis down")) await _cache_team_object( team_id="team-cache-outage", @@ -5188,10 +4971,7 @@ async def test_cache_team_object_tolerates_cache_invalidation_failures(): proxy_logging_obj=logging_obj, ) - written_keys = [ - (c.kwargs.get("key") or c.args[0]) - for c in cache.async_set_cache.await_args_list - ] + written_keys = [(c.kwargs.get("key") or c.args[0]) for c in cache.async_set_cache.await_args_list] assert written_keys == ["team_id:team-cache-outage"] @@ -5467,8 +5247,9 @@ async def test_common_checks_budget_reads_run_concurrently(): probe = _BudgetSpendConcurrencyProbe(expected=4) - with patch("litellm.proxy.proxy_server.prisma_client", None), patch( - "litellm.proxy.proxy_server.get_current_spend", probe + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch("litellm.proxy.proxy_server.get_current_spend", probe), ): task = asyncio.create_task( common_checks( @@ -5536,8 +5317,9 @@ async def test_common_checks_budget_gather_raises_highest_priority_scope(): request=MagicMock(spec=Request), ) - with patch("litellm.proxy.proxy_server.prisma_client", None), patch( - "litellm.proxy.proxy_server.get_current_spend", _spend_by_counter + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), ): # Both team and end-user over budget: team wins on priority. _spend_by_counter.team = 999.0 @@ -5572,8 +5354,9 @@ async def test_common_checks_personal_user_budget_blocks_in_gather(): async def _spend_by_counter(counter_key, fallback_spend, max_budget=None, **kwargs): return 999.0 if counter_key == "spend:user:u1" else 0.0 - with patch("litellm.proxy.proxy_server.prisma_client", None), patch( - "litellm.proxy.proxy_server.get_current_spend", _spend_by_counter + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), ): with pytest.raises(litellm.BudgetExceededError) as over: await common_checks( @@ -5615,9 +5398,11 @@ async def test_common_checks_personal_user_budget_skipped_for_team_key(): async def _no_membership(*args, **kwargs): return None - with patch("litellm.proxy.proxy_server.prisma_client", None), patch( - "litellm.proxy.proxy_server.get_current_spend", _spend_by_counter - ), patch("litellm.proxy.auth.auth_checks.get_team_membership", _no_membership): + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), + patch("litellm.proxy.auth.auth_checks.get_team_membership", _no_membership), + ): result = await common_checks( request_body={"messages": [{"role": "user", "content": "hi"}]}, team_object=team, @@ -5656,9 +5441,11 @@ async def test_common_checks_personal_user_budget_enforced_on_team_key_when_flag async def _no_membership(*args, **kwargs): return None - with patch("litellm.proxy.proxy_server.prisma_client", None), patch( - "litellm.proxy.proxy_server.get_current_spend", _spend_by_counter - ), patch("litellm.proxy.auth.auth_checks.get_team_membership", _no_membership): + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), + patch("litellm.proxy.auth.auth_checks.get_team_membership", _no_membership), + ): with pytest.raises(litellm.BudgetExceededError) as exc_info: await common_checks( request_body={"messages": [{"role": "user", "content": "hi"}]}, @@ -5689,8 +5476,9 @@ async def test_common_checks_personal_user_budget_still_enforced_on_personal_key async def _spend_by_counter(counter_key, fallback_spend, max_budget=None, **kwargs): return 999.0 if counter_key == "spend:user:u1" else 0.0 - with patch("litellm.proxy.proxy_server.prisma_client", None), patch( - "litellm.proxy.proxy_server.get_current_spend", _spend_by_counter + with ( + patch("litellm.proxy.proxy_server.prisma_client", None), + patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), ): with pytest.raises(litellm.BudgetExceededError): await common_checks( @@ -5773,10 +5561,11 @@ async def test_budget_checks_only_run_on_llm_api_routes(scope, route, expect_blo request=MagicMock(spec=Request), ) - with patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch( - "litellm.proxy.proxy_server.get_current_spend", _spend_by_counter - ), patch("litellm.proxy.auth.auth_checks.get_team_membership", _no_membership), patch( - "litellm.proxy.auth.auth_checks.get_org_object", _get_org + with ( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.get_current_spend", _spend_by_counter), + patch("litellm.proxy.auth.auth_checks.get_team_membership", _no_membership), + patch("litellm.proxy.auth.auth_checks.get_org_object", _get_org), ): if expect_blocked: with pytest.raises(litellm.BudgetExceededError): @@ -6275,9 +6064,7 @@ async def test_get_end_user_object_token_budget_gate_keeps_fetching_unrestricted mock_prisma = MagicMock() mock_prisma.db.litellm_endusertable.find_many = AsyncMock(return_value=[]) - mock_prisma.db.litellm_endusertable.find_unique = AsyncMock( - return_value=_end_user_db_row("eu-anon-1", spend=100.0) - ) + mock_prisma.db.litellm_endusertable.find_unique = AsyncMock(return_value=_end_user_db_row("eu-anon-1", spend=100.0)) cache = UserApiKeyCache() result = await get_end_user_object( @@ -6959,9 +6746,7 @@ async def test_common_checks_ignores_non_llm_route_when_enabled(monkeypatch): monkeypatch.setattr(litellm, "block_requests_for_models_without_pricing", True) router = _router_with_priced_and_unpriced_models() - result = await _run_common_checks( - model="unpriced-group", llm_router=router, route="/model/new" - ) + result = await _run_common_checks(model="unpriced-group", llm_router=router, route="/model/new") assert result is True @@ -7077,11 +6862,15 @@ def test_team_allowed_routes_exact_route_does_not_become_a_prefix_grant(): roles = LiteLLM_JWTAuth(team_allowed_routes=["/internal-models/model-a"]) assert ( - allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/internal-models/model-a", litellm_proxy_roles=roles) + allowed_routes_check( + user_role=LitellmUserRoles.TEAM, user_route="/internal-models/model-a", litellm_proxy_roles=roles + ) is True ) assert ( - allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/internal-models/model-b", litellm_proxy_roles=roles) + allowed_routes_check( + user_role=LitellmUserRoles.TEAM, user_route="/internal-models/model-b", litellm_proxy_roles=roles + ) is False ) @@ -7159,8 +6948,7 @@ async def test_invalidate_team_member_spend_state_sets_the_spend_counter_and_cle assert await real_cache.async_get_cache(key="team_membership:user-1:team-1") is None assert real_spend_counter_cache.in_memory_cache.get_cache(key="spend:team_member:user-1:team-1") == 0.0 assert ( - real_spend_counter_cache.in_memory_cache.get_cache(key="spend_db_floor:spend:team_member:user-1:team-1") - == 0.0 + real_spend_counter_cache.in_memory_cache.get_cache(key="spend_db_floor:spend:team_member:user-1:team-1") == 0.0 ), "the DB-floor marker kept the pre-reset value; a stale-floor read can raise the counter right back up" @@ -7356,9 +7144,9 @@ async def test_invalidate_team_member_spend_state_broadcasts_the_spend_counter_t ) assert remote_spend_counter_in_memory_cache.get_cache("spend:team_member:user-1:team-1") == 0.0 - assert ( - remote_spend_counter_in_memory_cache.get_cache("spend_db_floor:spend:team_member:user-1:team-1") == 0.0 - ), "the DB-floor marker was not broadcast; a remote worker can re-raise the counter off its stale floor" + assert remote_spend_counter_in_memory_cache.get_cache("spend_db_floor:spend:team_member:user-1:team-1") == 0.0, ( + "the DB-floor marker was not broadcast; a remote worker can re-raise the counter off its stale floor" + ) @pytest.mark.asyncio @@ -7541,3 +7329,81 @@ async def test_key_budget_error_keeps_the_masked_key_name(key_name): names are just as valid as the alphanumeric ones.""" message = await _run_key_budget_check(key_name) assert f"Key=prod-key ({key_name}) Current cost" in message + + +class _UntouchedPrisma: + def __getattr__(self, name: str) -> object: + raise AssertionError(f"database reached through {name}") + + +@pytest.mark.asyncio +async def test_enforced_model_allowlists_reads_every_level_from_cache(): + from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_ProjectTableCachedObj, + LiteLLM_TeamMembership, + LiteLLM_TeamTableCachedObj, + LiteLLM_UserTable, + ) + from litellm.proxy.auth.auth_checks import enforced_model_allowlists + from litellm.proxy.common_utils.user_api_key_cache import ( + UserApiKeyCache, + team_membership_reservation_cache_key, + ) + from litellm.proxy.utils import ProxyLogging + + cache = UserApiKeyCache() + proxy_logging_obj = ProxyLogging(user_api_key_cache=cache) + await cache.async_set_cache( + key="team_id:team-fake", value=LiteLLM_TeamTableCachedObj(team_id="team-fake", models=["gpt-4o"]) + ) + await cache.async_set_cache( + key=team_membership_reservation_cache_key(user_id="user-fake", team_id="team-fake"), + value=LiteLLM_TeamMembership( + user_id="user-fake", + team_id="team-fake", + litellm_budget_table=LiteLLM_BudgetTable(allowed_models=["gpt-4o-mini"]), + ), + ) + await cache.async_set_cache( + key="project_id:project-fake", + value=LiteLLM_ProjectTableCachedObj(project_id="project-fake", models=["gpt-4.1"]), + ) + await cache.async_set_cache(key="user-fake", value=LiteLLM_UserTable(user_id="user-fake", models=["o3"])) + prisma_client = _UntouchedPrisma() + + team_scoped = await enforced_model_allowlists( + valid_token=UserAPIKeyAuth( + token="hashed-fake", + models=["all-team-models"], + team_models=["gpt-4o", "gpt-4o-mini"], + user_id="user-fake", + team_id="team-fake", + project_id="project-fake", + ), + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=proxy_logging_obj, + ) + personal = await enforced_model_allowlists( + valid_token=UserAPIKeyAuth(token="hashed-fake", user_id="user-fake"), + prisma_client=prisma_client, + user_api_key_cache=cache, + proxy_logging_obj=proxy_logging_obj, + ) + without_database = await enforced_model_allowlists( + valid_token=UserAPIKeyAuth(token="hashed-fake", models=["gpt-4o"], user_id="user-fake", team_id="team-fake"), + prisma_client=None, + user_api_key_cache=cache, + proxy_logging_obj=proxy_logging_obj, + ) + + assert [list(scope) for scope in team_scoped] == [ + ["gpt-4o", "gpt-4o-mini"], + ["gpt-4o"], + ["gpt-4o-mini"], + [], + ["gpt-4.1"], + ] + assert [list(scope) for scope in personal] == [[], [], [], ["o3"], []] + assert [list(scope) for scope in without_database] == [["gpt-4o"]] diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py index 6578b75ace1..c96e7684e97 100644 --- a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -1,7 +1,7 @@ """OpenAI passthrough WebSocket route: registration, opt-in gating, and refusals.""" import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType, SimpleNamespace from typing import Final @@ -15,10 +15,13 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _OPENAI_WS_DISABLED_REFUSAL, _OPENAI_WS_MODEL_RESTRICTED_REFUSAL, _openai_websocket_refusal, + _proxy_model_allowlists, openai_websocket_proxy_route, router, ) +Scopes = tuple[Sequence[str], ...] + ENABLED: Final = MappingProxyType({"enable_openai_websocket_passthrough": True}) DISABLED_SETTINGS: Final = ( MappingProxyType({}), @@ -96,21 +99,39 @@ class _FakeRelay: ) +class _FakeModelAllowlists: + def __init__(self, scopes: Scopes) -> None: + self.scopes = scopes + self.calls: list[UserAPIKeyAuth] = [] + + async def __call__(self, valid_token: UserAPIKeyAuth, /) -> Scopes: + self.calls.append(valid_token) + return self.scopes + + +@dataclass(frozen=True, slots=True) +class _Served: + relay: _FakeRelay + allowlists: _FakeModelAllowlists + + async def _serve( websocket: _FakeWebSocket, endpoint: str, user_api_key_dict: UserAPIKeyAuth, general_settings: Mapping[str, object], -) -> _FakeRelay: - relay = _FakeRelay() + scopes: Scopes = (), +) -> _Served: + served = _Served(relay=_FakeRelay(), allowlists=_FakeModelAllowlists(scopes)) await openai_websocket_proxy_route( websocket=websocket, endpoint=endpoint, user_api_key_dict=user_api_key_dict, general_settings=general_settings, - relay=relay, + relay=served.relay, + model_allowlists=served.allowlists, ) - return relay + return served @pytest.mark.asyncio @@ -120,9 +141,9 @@ async def test_openai_websocket_forwards_query_and_keeps_provider_auth(prefix, m websocket = _FakeWebSocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview") with patch(GET_CREDENTIALS, return_value="sk-provider"): - relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) + served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) - assert relay.calls == [ + assert served.relay.calls == [ _RelayCall( target="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview", custom_headers=MappingProxyType({"Authorization": "Bearer sk-provider"}), @@ -145,10 +166,10 @@ async def test_openai_websocket_accepts_first_client_subprotocol(): ) with patch(GET_CREDENTIALS, return_value="sk-provider"): - relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) + served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) assert websocket.accepts == ["realtime"] - assert [call.accept_websocket for call in relay.calls] == [False] + assert [call.accept_websocket for call in served.relay.calls] == [False] assert websocket.closed is None @@ -157,13 +178,13 @@ async def test_openai_websocket_closes_cleanly_when_provider_credentials_missing websocket = _FakeWebSocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") with patch(GET_CREDENTIALS, return_value=None): - relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) + served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), ENABLED) assert websocket.closed is not None assert websocket.closed[0] == 1011 assert "OPENAI_API_KEY" in websocket.closed[1] assert websocket.accepts == [] - assert relay.calls == [] + assert served.relay.calls == [] @pytest.mark.asyncio @@ -172,23 +193,26 @@ async def test_openai_websocket_closes_cleanly_when_provider_credentials_missing async def test_openai_websocket_refused_unless_explicitly_enabled(prefix, general_settings): websocket = _FakeWebSocket(f"/{prefix}/v1/realtime", "model=gpt-4o-realtime-preview") - relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), general_settings) + served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), general_settings) assert "enable_openai_websocket_passthrough" in websocket.error_message() assert websocket.accepts == [None] assert websocket.closed == (1008, _OPENAI_WS_DISABLED_REFUSAL.close_reason) - assert relay.calls == [] + assert served.relay.calls == [] +@pytest.mark.asyncio @pytest.mark.parametrize("general_settings", DISABLED_SETTINGS) -def test_openai_websocket_refusal_is_disabled_for_falsy_settings(general_settings): - assert _openai_websocket_refusal(UserAPIKeyAuth(), general_settings) is _OPENAI_WS_DISABLED_REFUSAL +async def test_openai_websocket_refusal_is_disabled_for_falsy_settings(general_settings): + refusal = await _openai_websocket_refusal(UserAPIKeyAuth(), general_settings, _FakeModelAllowlists(())) + assert refusal is _OPENAI_WS_DISABLED_REFUSAL +@pytest.mark.asyncio @pytest.mark.parametrize("value", [True, "true", "True"]) -def test_openai_websocket_refusal_is_none_for_truthy_settings(value): +async def test_openai_websocket_refusal_is_none_for_truthy_settings(value): settings = MappingProxyType({"enable_openai_websocket_passthrough": value}) - assert _openai_websocket_refusal(UserAPIKeyAuth(), settings) is None + assert await _openai_websocket_refusal(UserAPIKeyAuth(), settings, _FakeModelAllowlists(())) is None @pytest.mark.asyncio @@ -199,51 +223,73 @@ async def test_openai_websocket_refusal_echoes_requested_subprotocol(): subprotocols="realtime, openai-beta.realtime-v1", ) - relay = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), MappingProxyType({})) + served = await _serve(websocket, "v1/realtime", UserAPIKeyAuth(), MappingProxyType({})) assert websocket.accepts == ["realtime"] assert websocket.closed == (1008, _OPENAI_WS_DISABLED_REFUSAL.close_reason) - assert relay.calls == [] + assert served.relay.calls == [] -RESTRICTED_KEYS: Final = ( - UserAPIKeyAuth(models=["gpt-4o"]), - UserAPIKeyAuth(team_models=["gpt-4o-realtime-preview"]), - UserAPIKeyAuth(models=["all-team-models"], team_models=["gpt-4o"]), +RESTRICTED_SCOPES: Final[tuple[Scopes, ...]] = ( + (("gpt-4o",),), + ((), ("gpt-4o-realtime-preview",)), + (("all-team-models",), ("gpt-4o",)), + ((), ("all-proxy-models",), ("gpt-4o",)), + ((), (), (), ("gpt-4o",)), + (("*",), (), (), (), ("gpt-4o",)), ) -UNRESTRICTED_KEYS: Final = ( - UserAPIKeyAuth(), - UserAPIKeyAuth(models=["all-proxy-models"]), - UserAPIKeyAuth(models=["*"]), - UserAPIKeyAuth(models=["all-team-models"], team_models=["all-proxy-models"]), +UNRESTRICTED_SCOPES: Final[tuple[Scopes, ...]] = ( + (), + ((),), + (("all-proxy-models",),), + (("*",),), + (("all-team-models",), ("all-proxy-models",)), + ((), (), (), (), ()), + (("*",), ("all-proxy-models",), ("all-team-models",), (), ()), ) @pytest.mark.asyncio -@pytest.mark.parametrize("user_api_key_dict", RESTRICTED_KEYS) -async def test_openai_websocket_rejects_model_restricted_keys(user_api_key_dict): +@pytest.mark.parametrize("scopes", RESTRICTED_SCOPES) +async def test_openai_websocket_rejects_model_restricted_identities(scopes): websocket = _FakeWebSocket("/openai/v1/realtime", "model=gpt-4o-realtime-preview") + user_api_key_dict = UserAPIKeyAuth(token="hashed-fake", user_id="user-fake", team_id="team-fake") - relay = await _serve(websocket, "v1/realtime", user_api_key_dict, ENABLED) + served = await _serve(websocket, "v1/realtime", user_api_key_dict, ENABLED, scopes) assert "model restrictions" in websocket.error_message() assert websocket.closed == (1008, _OPENAI_WS_MODEL_RESTRICTED_REFUSAL.close_reason) - assert relay.calls == [] - - -@pytest.mark.parametrize("user_api_key_dict", RESTRICTED_KEYS) -def test_openai_websocket_refusal_prefers_disabled_over_model_restriction(user_api_key_dict): - assert _openai_websocket_refusal(user_api_key_dict, MappingProxyType({})) is _OPENAI_WS_DISABLED_REFUSAL + assert served.relay.calls == [] + assert served.allowlists.calls == [user_api_key_dict] @pytest.mark.asyncio -@pytest.mark.parametrize("user_api_key_dict", UNRESTRICTED_KEYS) -async def test_openai_websocket_allows_unrestricted_keys(user_api_key_dict): +@pytest.mark.parametrize("scopes", RESTRICTED_SCOPES) +async def test_openai_websocket_disabled_refusal_skips_allowlist_lookups(scopes): + allowlists = _FakeModelAllowlists(scopes) + + refusal = await _openai_websocket_refusal(UserAPIKeyAuth(), MappingProxyType({}), allowlists) + + assert refusal is _OPENAI_WS_DISABLED_REFUSAL + assert allowlists.calls == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scopes", UNRESTRICTED_SCOPES) +async def test_openai_websocket_allows_unrestricted_identities(scopes): websocket = _FakeWebSocket("/openai/v1/responses", "") with patch(GET_CREDENTIALS, return_value="sk-provider"): - relay = await _serve(websocket, "v1/responses", user_api_key_dict, ENABLED) + served = await _serve(websocket, "v1/responses", UserAPIKeyAuth(), ENABLED, scopes) - assert len(relay.calls) == 1 + assert len(served.relay.calls) == 1 assert websocket.sent == [] assert websocket.closed is None + + +@pytest.mark.asyncio +async def test_proxy_model_allowlists_reads_the_key_scope_without_a_database(): + with patch("litellm.proxy.proxy_server.prisma_client", None): + scopes = await _proxy_model_allowlists()(UserAPIKeyAuth(models=["gpt-4o"])) + + assert tuple(tuple(scope) for scope in scopes) == (("gpt-4o",),) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index d91928a203e..34216f7e1b9 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -11568,14 +11568,10 @@ async def test_key_window_spend_row_is_enqueued_with_the_actual_cost(): reset_at = datetime.now(timezone.utc) + timedelta(days=10) key_obj = MagicMock() - key_obj.budget_limits = [ - {"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()} - ] + key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}] with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue: - await increment_spend_counters( - token="hashed-token", team_id=None, user_id=None, response_cost=0.25 - ) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) enqueued = await _drain(queue) assert len(enqueued) == 1 @@ -11594,14 +11590,10 @@ async def test_team_window_spend_row_is_enqueued(): reset_at = datetime.now(timezone.utc) + timedelta(days=3) team_obj = MagicMock() - team_obj.budget_limits = [ - {"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()} - ] + team_obj.budget_limits = [{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}] with _window_spend_enqueue_env({"team_id:team-1": team_obj}) as queue: - await increment_spend_counters( - token=None, team_id="team-1", user_id=None, response_cost=1.5 - ) + await increment_spend_counters(token=None, team_id="team-1", user_id=None, response_cost=1.5) enqueued = await _drain(queue) assert len(enqueued) == 1 @@ -11620,9 +11612,7 @@ async def test_window_spend_row_is_enqueued_even_when_the_counter_was_reserved() reset_at = datetime.now(timezone.utc) + timedelta(days=10) key_obj = MagicMock() - key_obj.budget_limits = [ - {"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()} - ] + key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}] reservation = { "entries": [ {"counter_key": "spend:key:hashed-token", "reserved": 1.0}, @@ -11660,9 +11650,7 @@ async def test_sliding_window_without_reset_at_is_not_enqueued(): key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0}] with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue: - await increment_spend_counters( - token="hashed-token", team_id=None, user_id=None, response_cost=0.25 - ) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) enqueued = await _drain(queue) assert enqueued == [] @@ -11680,9 +11668,7 @@ async def test_each_configured_window_gets_its_own_row_enqueue(): ] with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue: - await increment_spend_counters( - token="hashed-token", team_id=None, user_id=None, response_cost=0.25 - ) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) enqueued = await _drain(queue) assert sorted(item["window_duration"] for item in enqueued) == ["1d", "30d"] @@ -11697,9 +11683,7 @@ async def test_no_window_spend_row_enqueued_without_budget_limits(): key_obj.budget_limits = None with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue: - await increment_spend_counters( - token="hashed-token", team_id=None, user_id=None, response_cost=0.25 - ) + await increment_spend_counters(token="hashed-token", team_id=None, user_id=None, response_cost=0.25) enqueued = await _drain(queue) assert enqueued == [] @@ -11713,9 +11697,7 @@ async def test_window_spend_row_carries_the_request_start_time(): reset_at = datetime.now(timezone.utc) + timedelta(days=10) key_obj = MagicMock() - key_obj.budget_limits = [ - {"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()} - ] + key_obj.budget_limits = [{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}] with _window_spend_enqueue_env({"hashed-token": key_obj}) as queue: await increment_spend_counters( @@ -11736,9 +11718,7 @@ async def test_team_window_spend_row_carries_the_request_start_time(): reset_at = datetime.now(timezone.utc) + timedelta(days=3) team_obj = MagicMock() - team_obj.budget_limits = [ - {"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()} - ] + team_obj.budget_limits = [{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}] with _window_spend_enqueue_env({"team_id:team-1": team_obj}) as queue: await increment_spend_counters( @@ -12050,7 +12030,6 @@ async def test_init_guardrails_in_db_snapshots_and_reconciles_under_guardrail_re assert not GUARDRAIL_RECONCILE_LOCK.locked() - @pytest.mark.asyncio async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeypatch): from litellm.proxy.prompts.prompt_registry import IN_MEMORY_PROMPT_REGISTRY @@ -12094,7 +12073,9 @@ async def test_init_prompts_in_db_reloads_rows_patched_on_another_worker(monkeyp await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) assert served_content() == "Begin every reply with AHOY" - prisma_client.db.litellm_prompttable.find_many = AsyncMock(return_value=[db_row("Begin every reply with HOWDY")]) + prisma_client.db.litellm_prompttable.find_many = AsyncMock( + return_value=[db_row("Begin every reply with HOWDY")] + ) await ProxyConfig()._init_prompts_in_db(prisma_client=prisma_client) assert served_content() == "Begin every reply with HOWDY" @@ -12547,3 +12528,40 @@ def test_disabling_docs_does_not_disable_other_routes(monkeypatch): assert client.get("/redoc").status_code == 404 assert client.get("/health/liveliness").status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "db_general_settings, expected", + [ + ({"enable_openai_websocket_passthrough": True}, True), + ({"enable_openai_websocket_passthrough": False}, False), + ({}, None), + ], +) +async def test_update_general_settings_propagates_openai_websocket_passthrough(db_general_settings, expected): + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + + with patch("litellm.proxy.proxy_server.general_settings", {"enable_openai_websocket_passthrough": True}): + await proxy_config._update_general_settings(db_general_settings=db_general_settings) + + import litellm.proxy.proxy_server as ps + + assert ps.general_settings["enable_openai_websocket_passthrough"] is expected + + +@pytest.mark.asyncio +async def test_update_general_settings_keeps_yaml_openai_websocket_passthrough(): + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config = ProxyConfig() + proxy_config._yaml_general_settings_keys = {"enable_openai_websocket_passthrough"} + + with patch("litellm.proxy.proxy_server.general_settings", {"enable_openai_websocket_passthrough": False}): + await proxy_config._update_general_settings(db_general_settings={"enable_openai_websocket_passthrough": True}) + + import litellm.proxy.proxy_server as ps + + assert ps.general_settings["enable_openai_websocket_passthrough"] is False From 2a7fc8de01c2dca87de9cfc9514a91fddd7a43c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:05:35 -0700 Subject: [PATCH 3/9] fix(proxy): keep the token's team model list in the websocket passthrough gate without a database --- litellm/proxy/auth/auth_checks.py | 2 +- tests/test_litellm/proxy/auth/test_auth_checks.py | 10 ++++++++-- .../proxy/test_openai_ws_passthrough_routes.py | 9 ++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 98d334ce2cc..120a0bb29ea 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4164,7 +4164,7 @@ async def enforced_model_allowlists( """One model allowlist per level that ``common_checks`` enforces on a request from this identity.""" key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token) if prisma_client is None: - return (key_models,) + return (key_models, tuple(valid_token.team_models or ())) team_object: Final = ( None if valid_token.team_id is None diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 45e10948267..5242a99c54c 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -7392,7 +7392,13 @@ async def test_enforced_model_allowlists_reads_every_level_from_cache(): proxy_logging_obj=proxy_logging_obj, ) without_database = await enforced_model_allowlists( - valid_token=UserAPIKeyAuth(token="hashed-fake", models=["gpt-4o"], user_id="user-fake", team_id="team-fake"), + valid_token=UserAPIKeyAuth( + token="hashed-fake", + models=["gpt-4o"], + team_models=["gpt-4o-mini"], + user_id="user-fake", + team_id="team-fake", + ), prisma_client=None, user_api_key_cache=cache, proxy_logging_obj=proxy_logging_obj, @@ -7406,4 +7412,4 @@ async def test_enforced_model_allowlists_reads_every_level_from_cache(): ["gpt-4.1"], ] assert [list(scope) for scope in personal] == [[], [], [], ["o3"], []] - assert [list(scope) for scope in without_database] == [["gpt-4o"]] + assert [list(scope) for scope in without_database] == [["gpt-4o"], ["gpt-4o-mini"]] diff --git a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py index c96e7684e97..7d79192b884 100644 --- a/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py +++ b/tests/test_litellm/proxy/test_openai_ws_passthrough_routes.py @@ -14,6 +14,7 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( _OPENAI_WS_DISABLED_REFUSAL, _OPENAI_WS_MODEL_RESTRICTED_REFUSAL, + _has_model_restrictions, _openai_websocket_refusal, _proxy_model_allowlists, openai_websocket_proxy_route, @@ -288,8 +289,10 @@ async def test_openai_websocket_allows_unrestricted_identities(scopes): @pytest.mark.asyncio -async def test_proxy_model_allowlists_reads_the_key_scope_without_a_database(): +async def test_proxy_model_allowlists_reads_the_token_scopes_without_a_database(): + token: Final = UserAPIKeyAuth(models=[], team_id="team-fake", team_models=["gpt-4o"]) with patch("litellm.proxy.proxy_server.prisma_client", None): - scopes = await _proxy_model_allowlists()(UserAPIKeyAuth(models=["gpt-4o"])) + scopes = await _proxy_model_allowlists()(token) - assert tuple(tuple(scope) for scope in scopes) == (("gpt-4o",),) + assert tuple(tuple(scope) for scope in scopes) == ((), ("gpt-4o",)) + assert _has_model_restrictions(scopes) From f846388bb1174b5cdb89a7f019825ad83b6d6e81 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:24:30 -0700 Subject: [PATCH 4/9] fix(proxy): treat a missing user row as unrestricted in the websocket passthrough gate --- litellm/proxy/auth/auth_checks.py | 23 ++++++++++++++--- .../proxy/auth/test_auth_checks.py | 25 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 120a0bb29ea..b0e22153401 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -4155,6 +4155,24 @@ async def _granted_model_lists( ) +async def _user_object_or_none( + valid_token: UserAPIKeyAuth, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, +) -> LiteLLM_UserTable | None: + try: + return await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=proxy_logging_obj, + ) + except ValueError: + return None + + async def enforced_model_allowlists( valid_token: UserAPIKeyAuth, prisma_client: PrismaClient | None, @@ -4178,11 +4196,10 @@ async def enforced_model_allowlists( user_object: Final = ( None if team_object is not None - else await get_user_object( - user_id=valid_token.user_id, + else await _user_object_or_none( + valid_token=valid_token, prisma_client=prisma_client, user_api_key_cache=user_api_key_cache, - user_id_upsert=False, proxy_logging_obj=proxy_logging_obj, ) ) diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 5242a99c54c..7351c981838 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -7336,6 +7336,31 @@ class _UntouchedPrisma: raise AssertionError(f"database reached through {name}") +class _MissingUserPrisma: + class db: + class litellm_usertable: + @staticmethod + async def find_unique(where: dict[str, str], include: dict[str, bool]) -> None: + return None + + +@pytest.mark.asyncio +async def test_enforced_model_allowlists_treats_a_missing_user_row_as_unrestricted(): + from litellm.proxy.auth.auth_checks import enforced_model_allowlists + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + + cache = UserApiKeyCache() + scopes = await enforced_model_allowlists( + valid_token=UserAPIKeyAuth(token="hashed-fake", user_id="default_user_id"), + prisma_client=_MissingUserPrisma(), + user_api_key_cache=cache, + proxy_logging_obj=ProxyLogging(user_api_key_cache=cache), + ) + + assert [list(scope) for scope in scopes] == [[], [], [], [], []] + + @pytest.mark.asyncio async def test_enforced_model_allowlists_reads_every_level_from_cache(): from litellm.proxy._types import ( From a0b2e7fca6c0dd2bd22e81906eea41d2e2c87426 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:35:08 -0700 Subject: [PATCH 5/9] fix(proxy): only a provably missing user row counts as unrestricted in the websocket passthrough gate --- litellm/proxy/auth/auth_checks.py | 15 +++++++++--- .../proxy/auth/test_auth_checks.py | 24 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index b0e22153401..dc693317de0 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -2352,6 +2352,13 @@ async def _backfill_null_user_email( return updated_row +class UserNotFoundError(ValueError): + """The user row is provably absent, as opposed to merely unreadable, so a caller that reads a missing row as no user-level limits can key on it without also swallowing a database that would not answer.""" + + def __init__(self, user_id: str) -> None: + super().__init__(f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call.") + + @log_db_metrics async def get_user_object( user_id: str | None, @@ -2457,7 +2464,7 @@ async def get_user_object( value=None, last_db_access_time=last_db_access_time, ) - raise Exception + raise UserNotFoundError(user_id=user_id) if response.organization_memberships is not None and len(response.organization_memberships) > 0: # dump each organization membership to type LiteLLM_OrganizationMembershipTable @@ -2493,7 +2500,9 @@ async def get_user_object( ) return _response - except Exception as e: # if user not in db + except UserNotFoundError: + raise + except Exception as e: _log_budget_lookup_failure("user", e) raise ValueError( f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call. Got error - {e}" @@ -4169,7 +4178,7 @@ async def _user_object_or_none( user_id_upsert=False, proxy_logging_obj=proxy_logging_obj, ) - except ValueError: + except UserNotFoundError: return None diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 7351c981838..2284a05b2e9 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -7361,6 +7361,30 @@ async def test_enforced_model_allowlists_treats_a_missing_user_row_as_unrestrict assert [list(scope) for scope in scopes] == [[], [], [], [], []] +class _UnreachableUserPrisma: + class db: + class litellm_usertable: + @staticmethod + async def find_unique(where: dict[str, str], include: dict[str, bool]) -> None: + raise RuntimeError("database gone") + + +@pytest.mark.asyncio +async def test_enforced_model_allowlists_surfaces_a_failed_user_lookup(): + from litellm.proxy.auth.auth_checks import enforced_model_allowlists + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + + cache = UserApiKeyCache() + with pytest.raises(ValueError, match="database gone"): + await enforced_model_allowlists( + valid_token=UserAPIKeyAuth(token="hashed-fake", user_id="user-fake"), + prisma_client=_UnreachableUserPrisma(), + user_api_key_cache=cache, + proxy_logging_obj=ProxyLogging(user_api_key_cache=cache), + ) + + @pytest.mark.asyncio async def test_enforced_model_allowlists_reads_every_level_from_cache(): from litellm.proxy._types import ( From fafd294878fa7d7de600d70ed58906e5c0c900d2 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Fri, 4 Sep 2026 23:52:33 -0700 Subject: [PATCH 6/9] fix(mcp): let config.yaml MCP servers pin server_id (#39286) * fix(mcp): let config.yaml MCP servers pin server_id A config-defined MCP server's id is a hash of server_name|url|transport| auth_type|alias, recomputed on every config load, so editing any of those fields mints a new id. Every key and team granted the old id via object_permission.mcp_servers keeps pointing at an id that no longer exists, and the server disappears from tools/list for them with nothing logged. load_servers_from_config now uses an explicit server_id from the server's config entry when present and falls back to the existing hash otherwise, so grants survive url/name/alias edits. Rejected at config load: a blank or non-string server_id, two entries claiming the same id, a pinned id already held by a database-backed server, and a pinned id that is another entry's server_name or alias (expand_permission_list matches ids before names, so that one would capture the other server's grants). Because the database registry loads after the config on startup, a database row that lands on a pinned config id is reported as a warning from the database reload instead, where it is decidable; the warning is latched on the shadowed set so the config-reload timer does not reprint it every interval. Deployments that do not set server_id keep the exact id they have today. * fix(mcp): close two more pinned-id capture paths A pinned server_id equal to an alias supplied through litellm_settings mcp_aliases was accepted, because the collision index only held the entry's own alias field. expand_permission_list matches ids before names, so grants written for the aliased server resolved to the pinning one. mcp_aliases keys whose target is a config server are now reserved the same way. A pinned server_id equal to a database-backed server's name, server_name or alias had the same effect against the database side, and could not be rejected at config load because the database registry is not loaded yet. The database reload now warns about it, latched like the existing shadow warning. * fix(mcp): reserve only the aliases the loader actually assigns Reserving every mcp_aliases key targeting a config server was too broad in two ways: the mapping is ignored when the entry sets its own alias, and only the first mapping for a server is ever applied. Both cases made a pinned server_id that could never have collided abort proxy startup. Reserve only the name load_servers_from_config will really assign. The database capture warning also fired for a database server whose own id is the config server_id. There the database row wins the id outright through get_registry precedence, so the shadow warning above it is the accurate one and the capture message contradicted it. Skip those rows. Also mark the two litellm-internal patches in the reload test helper, which the test-quality gate counts; the database reload has no other seam. * fix(mcp): match the loader's alias check exactly, is None not falsiness load_servers_from_config consults mcp_aliases only when the entry has no alias key at all, so an entry setting alias: "" gets no mapped alias. The collision index used falsiness and reserved the mapped name anyway, which failed startup on a pinned server_id that could never have collided with it. * fix(mcp): skip one identifier, not the whole database row A database row can shadow one config server_id by id and capture another by name at the same time. Skipping the entire row when its id shadowed a config entry dropped the second warning, leaving the operator with half a diagnosis. Skip only the identifier equal to the row's own id. * fix(mcp): reject conflicting self-pinned server ids * fix(mcp): validate config server names before building the identifier index The collision check reads every entry's body up front, so a malformed entry under an invalid name surfaced as an AttributeError instead of the name validation error the loader gave before this change. --- .../mcp_server/mcp_server_manager.py | 223 ++++++- .../mcp_server/test_mcp_server_manager.py | 563 ++++++++++++++++++ 2 files changed, 782 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index bfc5f629faf..bcbcc6bc579 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -13,9 +13,19 @@ import json import os import re import time -from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence +from collections.abc import ( + AsyncIterator, + Awaitable, + Callable, + Container, + Iterable, + Mapping, + MutableMapping, + Sequence, +) from contextlib import asynccontextmanager from dataclasses import dataclass, replace +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypedDict, cast from urllib.parse import ParseResult, urlparse @@ -307,6 +317,7 @@ class MCPServerConfig(TypedDict, total=False): :meth:`MCPServerManager.load_servers_from_config`. Every key is optional: YAML supplies whatever the admin wrote, and each read applies its own default.""" + server_id: ReadOnly[str] alias: str description: str mcp_info: MCPInfo @@ -400,6 +411,164 @@ def _blank_to_none(value: str | None) -> str | None: return value.strip() or None +def _pinned_config_server_id(raw_server_id: object, server_name: str) -> str | None: + """Return the ``server_id`` an admin pinned for this config.yaml server, or ``None`` when absent. + + Without a pin the id is derived by hashing ``server_name|url|transport|auth_type|alias``, so + editing any of those fields mints a new id and every ``object_permission.mcp_servers`` grant + holding the old one silently stops matching. A pinned id is used verbatim and survives those + edits. Blank and non-string values are rejected rather than silently falling back to the hash, + because a config that pins an id and still churns is the failure this field exists to prevent. + + Under ``LITELLM_USE_SHORT_MCP_TOOL_PREFIX`` the tool prefix is derived from the server_id, so + pinning an id other than the one already in use renames every tool that server exposes. + """ + if raw_server_id is None: + return None + if not isinstance(raw_server_id, str) or not raw_server_id.strip(): + raise ValueError( + f"Invalid config for MCP server '{server_name}': server_id must be a non-empty string " + f"(got {raw_server_id!r})." + ) + return raw_server_id.strip() + + +def _first_mapped_alias(server_name: str, mcp_aliases: Mapping[str, str] | None) -> str | None: + """The ``mcp_aliases`` name ``load_servers_from_config`` will assign to this server, if any. + + Mirrors that loop, which takes the first mapping pointing at the server and stops. A later + mapping for the same server is never applied, so it stays free for another entry to pin. + """ + if mcp_aliases is None: + return None + return next( + (alias_name for alias_name, target_server_name in mcp_aliases.items() if target_server_name == server_name), + None, + ) + + +def _assigned_alias( + server_name: str, server_config: MCPServerConfig, mcp_aliases: Mapping[str, str] | None +) -> str | None: + """The alias ``load_servers_from_config`` will give this entry: its own, else the first mapping. + + ``is None``, not falsiness: the loader only consults the mapping when the key is absent, so an + entry that sets ``alias: ""`` gets no mapped alias and reserves nothing. + """ + alias: Final = server_config.get("alias") + return _first_mapped_alias(server_name, mcp_aliases) if alias is None else alias + + +def _validate_config_server_names(mcp_servers_config: Mapping[str, MCPServerConfig]) -> None: + """Reject bad server names before ``_config_identifier_owners`` reads any entry's body. + + The identifier index walks every entry up front, so without this pass a malformed entry under + a bad name would surface as an ``AttributeError`` from the index instead of the name error. + """ + for server_name in mcp_servers_config: + validate_mcp_server_name(server_name) + + +def _config_identifier_owners( + mcp_servers_config: Mapping[str, MCPServerConfig], + mcp_aliases: Mapping[str, str] | None, +) -> Mapping[str, frozenset[str]]: + """Map every server_name and alias in the config to the entries that own it. + + ``expand_permission_list`` resolves a grant against the registry keys before it falls back to + matching alias and server_name, so an id equal to another entry's name or alias captures that + entry's grants. Derived ids are hashes and never collide with a name, so this only matters once + an id is pinned. + + An alias is either set on the entry or mapped to it from ``litellm_settings.mcp_aliases``. Only + a name the loader below will really assign is reserved: the mapping is ignored for an entry that + sets its own ``alias``, and only the first mapping wins for one that does not, so reserving every + mapping would fail startup on a pin that was never going to collide. + + One identifier can have several owners when an entry's alias equals another entry's name. All of + them are kept: a grant naming that identifier resolves to every match while no id is pinned, and + a pin equal to it would narrow the grant to the pinning entry alone, even when that entry is one + of the owners. + """ + claims: Final = tuple( + (identifier, server_name) + for server_name, server_config in mcp_servers_config.items() + for identifier in (server_name, _assigned_alias(server_name, server_config, mcp_aliases)) + if identifier + ) + return MappingProxyType( + {identifier: frozenset(owner for claimed, owner in claims if claimed == identifier) for identifier, _ in claims} + ) + + +def _config_ids_capturing_db_identifiers( + config_server_ids: Container[str], + db_servers: Iterable[MCPServer], +) -> frozenset[str]: + """Config server ids that are a database-backed server's name, server_name or alias. + + ``expand_permission_list`` matches a grant against the registry keys before it matches names, so + such an id answers every grant written for the database server, and the database server itself + stops being reachable by name. The config load cannot catch this because the database registry + is not loaded yet, so it is reported from the reload that does have both halves. + + An identifier equal to the database server's own id is skipped: ``get_registry`` is + ``config_mcp_servers | registry``, so there the database server wins the id outright and the + shadow warning above is the accurate one. Reporting both would contradict. The skip is per + identifier rather than per server, so a row that shadows one config id and captures another + still reports the capture. + """ + return frozenset( + identifier + for server in db_servers + for identifier in (server.name, server.server_name, server.alias) + if identifier and identifier != server.server_id and identifier in config_server_ids + ) + + +def _reject_config_server_id_collision( + assigned_server_ids: Mapping[str, str], + server_id: str, + server_name: str, + pinned: bool, + db_backed_server_ids: Mapping[str, object], + identifier_owners: Mapping[str, frozenset[str]], +) -> None: + """Raise when ``server_id`` is already taken, either by an earlier config entry or by the database. + + Two config entries sharing an id would silently overwrite each other in ``config_mcp_servers``, + and an id already held by a database-backed server is hidden by it, because ``get_registry`` is + ``config_mcp_servers | registry`` and the right operand wins. A pinned id that is another + entry's server_name or alias captures that entry's permission grants the same way. Derived ids + cannot collide (the unique config key is part of the hash input), so all three only happen once + an id is pinned. + + Pinning an identifier this entry itself owns is allowed, because a grant naming it already + resolved here, but only when no other entry owns it too. An entry whose alias is this entry's + server_name shares the identifier, and pinning it would take that entry's grants. + """ + claimed_by = assigned_server_ids.get(server_id) + if claimed_by is not None: + raise ValueError( + f"Invalid config for MCP server '{server_name}': server_id '{server_id}' is already " + f"used by MCP server '{claimed_by}'. Each mcp_servers entry needs its own id." + ) + if pinned and server_id in db_backed_server_ids: + raise ValueError( + f"Invalid config for MCP server '{server_name}': server_id '{server_id}' belongs to a " + "database-backed MCP server. The database entry takes precedence over config.yaml, so " + "this server would never be reachable." + ) + other_owners: Final = identifier_owners.get(server_id, frozenset()) - frozenset((server_name,)) + if pinned and other_owners: + owner_names: Final = "', '".join(sorted(other_owners)) + raise ValueError( + f"Invalid config for MCP server '{server_name}': server_id '{server_id}' is the " + f"server_name or alias of MCP server '{owner_names}'. Permission entries naming " + f"'{server_id}' would resolve to '{server_name}' alone and no longer reach '{owner_names}'." + ) + + def _uses_issuer_anchor(manual_issuer: str | None, is_discovery_auth_type: bool) -> bool: """Whether the endpoints are authoritatively anchored to an admin-pinned issuer (RFC 8414 §3.3). @@ -1565,6 +1734,11 @@ class MCPServerManager: # empty result, or failure). Used to throttle re-probes for servers that do # not return instructions, and to apply a short cooldown after failures. self._upstream_initialize_instructions_probed_at: dict[str, float] = {} + # Last set of config server ids found shadowed by database rows. reload_servers_from_database + # runs on the config-reload timer, so this keeps a standing misconfiguration from re-logging + # the same warning every interval; a change in the set logs again. + self._warned_shadowed_config_server_ids: frozenset[str] = frozenset() + self._warned_capturing_config_server_ids: frozenset[str] = frozenset() self._oauth_discovery_on_startup = _mcp_oauth_discovery_on_startup_enabled() self._oauth_discovery_generation_counter = 0 self._oauth_discovery_slots: tuple[_OAuthDiscoverySlot, ...] = () @@ -1958,10 +2132,14 @@ class MCPServerManager: # Track which aliases have been used to ensure only first occurrence is used used_aliases: Final = set() + # server_id -> the config server_name that claimed it, so a pinned id cannot silently + # overwrite another server's entry in self.config_mcp_servers. + assigned_server_ids: MutableMapping[str, str] = {} # mutable-ok: per-load collision index + _validate_config_server_names(mcp_servers_config) + identifier_owners: Final = _config_identifier_owners(mcp_servers_config, mcp_aliases) for server_name, raw_server_config in mcp_servers_config.items(): server_config: MCPServerConfig = raw_server_config - validate_mcp_server_name(server_name) _mcp_info: MCPInfo = server_config.get("mcp_info", None) or {} # Preserve all custom fields from config while setting defaults for core fields mcp_info: MCPInfo = _mcp_info.copy() @@ -1994,14 +2172,24 @@ class MCPServerManager: name_for_prefix = get_server_prefix(temp_server) server_url = server_config.get("url", None) or "" - # Generate stable server ID based on parameters - server_id = self._generate_stable_server_id( + # An explicitly pinned server_id wins; otherwise derive one from the parameters. + pinned_server_id = _pinned_config_server_id(server_config.get("server_id"), server_name) + server_id = pinned_server_id or self._generate_stable_server_id( server_name=server_name, url=server_url, transport=server_config.get("transport", MCPTransport.http), auth_type=server_config.get("auth_type", None), alias=alias, ) + _reject_config_server_id_collision( + assigned_server_ids, + server_id, + server_name, + pinned=pinned_server_id is not None, + db_backed_server_ids=self.registry, + identifier_owners=identifier_owners, + ) + assigned_server_ids[server_id] = server_name _warn_on_server_name_fields( server_id=server_id, @@ -6123,6 +6311,33 @@ class MCPServerManager: verbose_logger.debug("MCP registry refreshed (%s servers in registry)", len(registered_registry)) + # get_registry() is ``config_mcp_servers | registry``, so a database row sharing an id with a + # config.yaml server hides that server everywhere. Only reachable once an operator pins + # ``server_id`` in config.yaml; say so rather than letting the server disappear silently. + shadowed_config_server_ids: Final = frozenset(self.config_mcp_servers.keys() & registered_registry.keys()) + if shadowed_config_server_ids and shadowed_config_server_ids != self._warned_shadowed_config_server_ids: + verbose_logger.warning( + "config.yaml MCP server_id(s) %s are also database-backed MCP servers. The database " + "entry takes precedence, so the config.yaml server is unreachable. Give the config " + "entry a different server_id.", + ", ".join(sorted(shadowed_config_server_ids)), + ) + self._warned_shadowed_config_server_ids = shadowed_config_server_ids + + # The mirror image of the block above: a config server_id that is a database server's name + # answers that server's grants instead, because ids are matched before names. + capturing_config_server_ids: Final = _config_ids_capturing_db_identifiers( + self.config_mcp_servers.keys(), registered_registry.values() + ) + if capturing_config_server_ids and capturing_config_server_ids != self._warned_capturing_config_server_ids: + verbose_logger.warning( + "config.yaml MCP server_id(s) %s are the name or alias of a database-backed MCP " + "server. Permission entries naming them resolve to the config.yaml server, not the " + "database one. Give the config entry a different server_id.", + ", ".join(sorted(capturing_config_server_ids)), + ) + self._warned_capturing_config_server_ids = capturing_config_server_ids + await self._hydrate_config_servers_dcr_clients() def get_mcp_servers_from_ids(self, server_ids: list[str]) -> list[MCPServer]: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 02b1a19081a..9745e508703 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -11428,6 +11428,569 @@ class TestOpenApiHandlerRelaysUpstreamAuth: assert "upstream returned HTTP 503" in result.content[0].text +class TestConfigServerIdPinning: + """config.yaml servers may pin ``server_id`` so permission grants survive connection edits.""" + + @staticmethod + def _config(**overrides: object) -> dict[str, dict[str, object]]: + return { + "docs_server": { + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + **overrides, + } + } + + @pytest.mark.asyncio + async def test_derived_id_churns_when_connection_fields_change(self): + """The behavior the pin exists to escape: editing the url mints a brand-new id.""" + manager = MCPServerManager() + + await manager.load_servers_from_config(self._config()) + before = next(iter(manager.config_mcp_servers)) + + manager.config_mcp_servers.clear() + await manager.load_servers_from_config(self._config(url="https://prod.example.com/mcp")) + after = next(iter(manager.config_mcp_servers)) + + assert before != after + + @pytest.mark.asyncio + async def test_pinned_id_survives_url_transport_auth_and_alias_edits(self): + manager = MCPServerManager() + + await manager.load_servers_from_config(self._config(server_id="docs-prod-1")) + assert list(manager.config_mcp_servers) == ["docs-prod-1"] + assert manager.config_mcp_servers["docs-prod-1"].server_id == "docs-prod-1" + + manager.config_mcp_servers.clear() + await manager.load_servers_from_config( + self._config( + server_id="docs-prod-1", + url="https://prod.example.com/mcp", + transport=MCPTransport.sse, + auth_type=MCPAuth.bearer_token, + alias="docs", + ) + ) + + assert list(manager.config_mcp_servers) == ["docs-prod-1"] + assert manager.config_mcp_servers["docs-prod-1"].url == "https://prod.example.com/mcp" + + @pytest.mark.asyncio + async def test_absent_server_id_keeps_the_derived_hash(self): + manager = MCPServerManager() + + await manager.load_servers_from_config(self._config()) + + derived = manager._generate_stable_server_id( + server_name="docs_server", + url="https://example.com/mcp", + transport=MCPTransport.http, + auth_type=None, + alias=None, + ) + assert list(manager.config_mcp_servers) == [derived] + + @pytest.mark.asyncio + @pytest.mark.parametrize("bad_value", ["", " ", 123, True, ["docs-prod-1"]]) + async def test_blank_or_non_string_server_id_is_rejected(self, bad_value: Any): + manager = MCPServerManager() + + with pytest.raises(ValueError, match="server_id must be a non-empty string"): + await manager.load_servers_from_config(self._config(server_id=bad_value)) + + @pytest.mark.asyncio + async def test_two_servers_pinning_the_same_id_are_rejected(self): + manager = MCPServerManager() + config: Dict[str, Any] = { + "docs_server": {"url": "https://a.example.com/mcp", "server_id": "shared-id"}, + "wiki_server": {"url": "https://b.example.com/mcp", "server_id": "shared-id"}, + } + + with pytest.raises(ValueError, match="already used by MCP server 'docs_server'"): + await manager.load_servers_from_config(config) + + @pytest.mark.asyncio + async def test_pinned_id_colliding_with_a_derived_id_is_rejected(self): + """A pin that lands on another entry's derived hash collides just as hard.""" + manager = MCPServerManager() + derived = manager._generate_stable_server_id( + server_name="docs_server", + url="https://a.example.com/mcp", + transport=MCPTransport.http, + auth_type=None, + alias=None, + ) + config: Dict[str, Any] = { + "docs_server": {"url": "https://a.example.com/mcp", "transport": MCPTransport.http}, + "wiki_server": {"url": "https://b.example.com/mcp", "server_id": derived}, + } + + with pytest.raises(ValueError, match="already used by MCP server 'docs_server'"): + await manager.load_servers_from_config(config) + + @pytest.mark.asyncio + async def test_pinned_id_colliding_with_a_db_backed_server_is_rejected(self): + """get_registry() is ``config | registry``, so the db row would hide the config server. + + The registry is seeded by hand because on a real startup the config loads before the + database does, so this check only fires on a later reload. The startup ordering is covered + by ``test_db_row_arriving_on_a_pinned_config_id_warns``; the warning there is not redundant. + """ + manager = MCPServerManager() + manager.registry["db-uuid-1"] = MCPServer( + server_id="db-uuid-1", + name="db_server", + transport=MCPTransport.http, + url="https://db.example.com/mcp", + ) + + with pytest.raises(ValueError, match="belongs to a database-backed MCP server"): + await manager.load_servers_from_config(self._config(server_id="db-uuid-1")) + + @pytest.mark.asyncio + async def test_derived_id_matching_a_db_backed_server_is_not_rejected(self): + """Only a pinned id is an authoring error; a hash collision must not fail startup.""" + manager = MCPServerManager() + derived = manager._generate_stable_server_id( + server_name="docs_server", + url="https://example.com/mcp", + transport=MCPTransport.http, + auth_type=None, + alias=None, + ) + manager.registry[derived] = MCPServer( + server_id=derived, + name="db_server", + transport=MCPTransport.http, + url="https://db.example.com/mcp", + ) + + await manager.load_servers_from_config(self._config()) + + assert derived in manager.config_mcp_servers + + @pytest.mark.asyncio + async def test_pinned_id_is_stripped_of_surrounding_whitespace(self): + manager = MCPServerManager() + + await manager.load_servers_from_config(self._config(server_id=" docs-prod-1 ")) + + assert list(manager.config_mcp_servers) == ["docs-prod-1"] + + @staticmethod + async def _reload_with_db_server(manager: MCPServerManager, server_id: str, db_name: str = "db_server") -> None: + row = LiteLLM_MCPServerTable( + server_id=server_id, + server_name=db_name, + alias=db_name, + url="https://db.example.com/mcp", + transport=MCPTransport.http, + ) + raw_row = MagicMock() + raw_row.model_dump.return_value = row.model_dump() + repository = MagicMock() + repository.table.find_many = AsyncMock(return_value=[raw_row]) + built = MCPServer( + server_id=server_id, + name=db_name, + server_name=db_name, + url="https://db.example.com/mcp", + transport=MCPTransport.http, + ) + with ( + patch( # test-quality-ok: the db reload path has no seam but its own repository + "litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPServerRepository", + return_value=repository, + ), + patch( # test-quality-ok: same, the prisma client is fetched inside the reload + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch.object(manager, "build_mcp_server_from_table", new=AsyncMock(return_value=built)), + ): + await manager.reload_servers_from_database() + + @pytest.mark.asyncio + async def test_db_row_arriving_on_a_pinned_config_id_warns(self, caplog): + """The db row loads after config on startup, so the config server is hidden then, not at load.""" + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="docs-prod-1")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "docs-prod-1") + + assert any("docs-prod-1" in m and "database entry takes precedence" in m for m in caplog.messages) + assert manager.get_registry()["docs-prod-1"].url == "https://db.example.com/mcp" + + @pytest.mark.asyncio + async def test_db_row_with_a_distinct_id_does_not_warn(self, caplog): + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="docs-prod-1")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "db-uuid-1") + + assert all("database entry takes precedence" not in m for m in caplog.messages) + assert set(manager.get_registry()) == {"docs-prod-1", "db-uuid-1"} + + @pytest.mark.asyncio + async def test_pinned_id_matching_another_entrys_server_name_is_rejected(self): + """expand_permission_list resolves against registry keys first, so this steals the grants.""" + manager = MCPServerManager() + + with pytest.raises(ValueError, match="server_name or alias of MCP server 'wiki_server'"): + await manager.load_servers_from_config( + { + "wiki_server": {"url": "https://wiki.example.com/mcp", "transport": MCPTransport.http}, + "docs_server": { + "server_id": "wiki_server", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + } + ) + + @pytest.mark.asyncio + async def test_pinned_id_matching_another_entrys_alias_is_rejected(self): + manager = MCPServerManager() + + with pytest.raises(ValueError, match="server_name or alias of MCP server 'wiki_server'"): + await manager.load_servers_from_config( + { + "wiki_server": { + "alias": "wiki", + "url": "https://wiki.example.com/mcp", + "transport": MCPTransport.http, + }, + "docs_server": { + "server_id": "wiki", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + } + ) + + @pytest.mark.asyncio + async def test_pinning_a_servers_own_name_is_allowed(self): + """The most natural pin an operator writes; it resolves to the same server either way.""" + manager = MCPServerManager() + + await manager.load_servers_from_config(self._config(server_id="docs_server")) + + assert list(manager.config_mcp_servers) == ["docs_server"] + + @pytest.mark.asyncio + async def test_pinning_a_servers_own_alias_is_allowed(self): + manager = MCPServerManager() + + await manager.load_servers_from_config(self._config(alias="docs", server_id="docs")) + + assert list(manager.config_mcp_servers) == ["docs"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("aliasing_entry_first", [True, False]) + async def test_pinning_own_name_that_is_another_entrys_alias_is_rejected(self, aliasing_entry_first: bool): + """A grant naming 'docs_server' reaches both servers unpinned; the pin would narrow it to one.""" + manager = MCPServerManager() + wiki = ( + "wiki_server", + {"alias": "docs_server", "url": "https://wiki.example.com/mcp", "transport": MCPTransport.http}, + ) + docs = ( + "docs_server", + {"server_id": "docs_server", "url": "https://example.com/mcp", "transport": MCPTransport.http}, + ) + + with pytest.raises(ValueError, match="server_name or alias of MCP server 'wiki_server'"): + await manager.load_servers_from_config(dict((wiki, docs) if aliasing_entry_first else (docs, wiki))) + + @pytest.mark.asyncio + async def test_pinning_own_name_that_is_another_entrys_mapped_alias_is_rejected(self): + manager = MCPServerManager() + + with pytest.raises(ValueError, match="server_name or alias of MCP server 'wiki_server'"): + await manager.load_servers_from_config( + { + "wiki_server": {"url": "https://wiki.example.com/mcp", "transport": MCPTransport.http}, + "docs_server": { + "server_id": "docs_server", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + }, + mcp_aliases={"docs_server": "wiki_server"}, + ) + + @pytest.mark.asyncio + async def test_pinning_own_alias_shared_with_a_later_entry_is_rejected(self): + """Nothing rejects duplicate aliases, so the first entry's pin would answer the second's grants.""" + manager = MCPServerManager() + + with pytest.raises(ValueError, match="server_name or alias of MCP server 'docs_server'"): + await manager.load_servers_from_config( + { + "wiki_server": { + "alias": "shared", + "server_id": "shared", + "url": "https://wiki.example.com/mcp", + "transport": MCPTransport.http, + }, + "docs_server": { + "alias": "shared", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + } + ) + + @pytest.mark.asyncio + async def test_own_name_pin_resolves_grants_like_the_unpinned_name(self): + """The negative control: a sole-owner self-pin must keep loading and answer the same grants.""" + manager = MCPServerManager() + + await manager.load_servers_from_config( + { + "wiki_server": {"alias": "wiki", "url": "https://wiki.example.com/mcp", "transport": MCPTransport.http}, + "docs_server": { + "server_id": "docs_server", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + } + ) + wiki_id = next(sid for sid, server in manager.config_mcp_servers.items() if server.alias == "wiki") + + assert manager.expand_permission_list(["docs_server"]) == ["docs_server"] + assert manager.expand_permission_list(["wiki"]) == [wiki_id] + + @pytest.mark.asyncio + async def test_derived_id_is_not_checked_against_names(self): + """Unpinned configs must keep loading; only a pinned id can be an authoring error.""" + manager = MCPServerManager() + + await manager.load_servers_from_config( + { + "wiki_server": {"url": "https://wiki.example.com/mcp", "transport": MCPTransport.http}, + "docs_server": {"url": "https://example.com/mcp", "transport": MCPTransport.http}, + } + ) + + assert len(manager.config_mcp_servers) == 2 + + @pytest.mark.asyncio + async def test_shadow_warning_is_not_repeated_on_every_reload(self, caplog): + """reload_servers_from_database runs on the config-reload timer; one warning, not one a tick.""" + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="docs-prod-1")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "docs-prod-1") + first_round = [m for m in caplog.messages if "database entry takes precedence" in m] + await self._reload_with_db_server(manager, "docs-prod-1") + second_round = [m for m in caplog.messages if "database entry takes precedence" in m] + + assert len(first_round) == 1 + assert second_round == first_round + + @pytest.mark.asyncio + async def test_shadow_warning_fires_again_when_the_shadowed_set_changes(self, caplog): + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="docs-prod-1")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "docs-prod-1") + await self._reload_with_db_server(manager, "db-uuid-1") + await self._reload_with_db_server(manager, "docs-prod-1") + + assert len([m for m in caplog.messages if "database entry takes precedence" in m]) == 2 + + @pytest.mark.asyncio + async def test_pinned_id_matching_a_mapped_alias_is_rejected(self): + """An alias can also arrive from litellm_settings.mcp_aliases; it is reserved just the same.""" + manager = MCPServerManager() + + with pytest.raises(ValueError, match="server_name or alias of MCP server 'wiki_server'"): + await manager.load_servers_from_config( + { + "wiki_server": {"url": "https://wiki.example.com/mcp", "transport": MCPTransport.http}, + "docs_server": { + "server_id": "wiki", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + }, + {"wiki": "wiki_server"}, + ) + + @pytest.mark.asyncio + async def test_pinning_a_servers_own_mapped_alias_is_allowed(self): + manager = MCPServerManager() + + await manager.load_servers_from_config( + self._config(server_id="docs"), + {"docs": "docs_server"}, + ) + + assert list(manager.config_mcp_servers) == ["docs"] + + @pytest.mark.asyncio + async def test_mapped_alias_for_an_unknown_server_reserves_nothing(self): + """A dangling mcp_aliases entry is never applied, so it must not fail an unrelated pin.""" + manager = MCPServerManager() + + await manager.load_servers_from_config( + self._config(server_id="wiki"), + {"wiki": "a_server_that_does_not_exist"}, + ) + + assert list(manager.config_mcp_servers) == ["wiki"] + + @pytest.mark.asyncio + async def test_config_id_that_is_a_db_server_name_warns(self, caplog): + """The mirror of the shadow case: here the config entry captures the db server's grants.""" + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="db_server")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "db-uuid-1") + + assert any("db_server" in m and "name or alias of a database-backed" in m for m in caplog.messages) + + @pytest.mark.asyncio + async def test_capture_warning_is_not_repeated_on_every_reload(self, caplog): + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="db_server")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "db-uuid-1") + await self._reload_with_db_server(manager, "db-uuid-1") + + assert len([m for m in caplog.messages if "name or alias of a database-backed" in m]) == 1 + + @pytest.mark.asyncio + async def test_config_id_unrelated_to_db_names_does_not_warn(self, caplog): + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="docs-prod-1")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "db-uuid-1") + + assert all("name or alias of a database-backed" not in m for m in caplog.messages) + + @pytest.mark.asyncio + async def test_mapped_alias_for_a_server_with_its_own_alias_reserves_nothing(self): + """load_servers_from_config ignores the mapping when the entry sets alias, so it is free.""" + manager = MCPServerManager() + + await manager.load_servers_from_config( + { + "wiki_server": { + "alias": "wiki_prod", + "url": "https://wiki.example.com/mcp", + "transport": MCPTransport.http, + }, + "docs_server": { + "server_id": "wiki", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + }, + {"wiki": "wiki_server"}, + ) + + assert "wiki" in manager.config_mcp_servers + assert len(manager.config_mcp_servers) == 2 + + @pytest.mark.asyncio + async def test_only_the_first_mapped_alias_for_a_server_is_reserved(self): + """Only the first mapping is applied, so pinning the second one must still load.""" + manager = MCPServerManager() + + await manager.load_servers_from_config( + { + "wiki_server": {"url": "https://wiki.example.com/mcp", "transport": MCPTransport.http}, + "docs_server": { + "server_id": "wiki_two", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + }, + {"wiki_one": "wiki_server", "wiki_two": "wiki_server"}, + ) + + assert "wiki_two" in manager.config_mcp_servers + + @pytest.mark.asyncio + async def test_invalid_name_is_reported_before_any_entry_body_is_read(self): + """The identifier index walks every entry up front, so a bad name must still fail on the name.""" + with pytest.raises(Exception, match="Server name cannot contain"): + await MCPServerManager().load_servers_from_config({"my-server": None}) + + @pytest.mark.asyncio + async def test_a_shadowing_db_server_reports_only_the_shadow_warning(self, caplog): + """The db row wins the id outright, so the capture message would contradict the shadow one.""" + manager = MCPServerManager() + await manager.load_servers_from_config(self._config(server_id="db_server")) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "db_server") + + assert any("database entry takes precedence" in m for m in caplog.messages) + assert all("name or alias of a database-backed" not in m for m in caplog.messages) + assert manager.get_registry()["db_server"].url == "https://db.example.com/mcp" + + @pytest.mark.asyncio + async def test_an_explicitly_blank_alias_still_blocks_the_mapping(self): + """The loader only consults mcp_aliases when the key is absent, so a blank alias frees it.""" + manager = MCPServerManager() + + await manager.load_servers_from_config( + { + "wiki_server": { + "alias": "", + "url": "https://wiki.example.com/mcp", + "transport": MCPTransport.http, + }, + "docs_server": { + "server_id": "wiki", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + }, + {"wiki": "wiki_server"}, + ) + + assert "wiki" in manager.config_mcp_servers + assert manager.config_mcp_servers["wiki"].url == "https://example.com/mcp" + + @pytest.mark.asyncio + async def test_a_row_that_shadows_one_id_still_reports_capturing_another(self, caplog): + """Skipping is per identifier, not per row, so the second collision is not lost.""" + manager = MCPServerManager() + await manager.load_servers_from_config( + { + "docs_server": { + "server_id": "shadow_x", + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + }, + "wiki_server": { + "server_id": "capture_y", + "url": "https://wiki.example.com/mcp", + "transport": MCPTransport.http, + }, + } + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + await self._reload_with_db_server(manager, "shadow_x", db_name="capture_y") + + assert any("shadow_x" in m and "database entry takes precedence" in m for m in caplog.messages) + assert any("capture_y" in m and "name or alias of a database-backed" in m for m in caplog.messages) + + class TestLitellmAdmissionKeyIsNeverTheSubjectToken: """The bearer that admitted the request as a LiteLLM key must not be sent to the IdP as the RFC 8693 subject_token (or ID-JAG assertion). Only ``x-litellm-api-key`` disambiguates: with it From f3cf5578989e558438b11c335b69702812d6b738 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 4 Sep 2026 23:57:53 -0700 Subject: [PATCH 7/9] feat(dashboard): configure classifier vision input (#39840) --- .../add_model/ClassificationMethodConfig.tsx | 10 ++- .../add_model/ClassifierVisionConfig.tsx | 79 ++++++++++++++++++ .../add_model/ComplexityRouterConfig.test.tsx | 81 +++++++++++++++++++ .../build_complexity_router_config.test.ts | 40 ++++++++- .../build_complexity_router_config.ts | 10 +-- .../edit_auto_router_modal.test.tsx | 57 +++++++++++++ 6 files changed, 268 insertions(+), 9 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/ClassifierVisionConfig.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx index cc66103fc86..15cfff01766 100644 --- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx @@ -15,6 +15,7 @@ import { RestrictedSection, restrictedBy } from "./TierRestrictions"; import HeuristicScoringConfig from "./HeuristicScoringConfig"; import ClassifierReasoningEffortSelect from "./ClassifierReasoningEffortSelect"; import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; +import ClassifierVisionConfig from "./ClassifierVisionConfig"; import type { ReasoningEffort } from "./complexity_router_tiers"; import { useComplexityScorerDefaults } from "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults"; import { @@ -315,12 +316,13 @@ const ClassificationMethodConfig: React.FC = ({ timeout_ms: value.classifier_llm_config?.timeout_ms ?? DEFAULT_CLASSIFIER_TIMEOUT_MS, classification_rubric: selectedRubric, }; - onChange({ + const nextValue: ComplexityRouterConfigValue = { ...value, ...(selectedRubric && { classifier_llm_config: rubricConfig }), classification_prompt: classificationPrompt, classification_examples: classificationExamples, - }); + }; + onChange(nextValue); }; const handleClassifierModelChange = (model: string) => { @@ -577,6 +579,10 @@ const ClassificationMethodConfig: React.FC = ({ value={value.classifier_llm_config ?? { model: "", timeout_ms: DEFAULT_CLASSIFIER_TIMEOUT_MS }} onChange={(classifier_llm_config) => onChange({ ...value, classifier_llm_config })} /> + onChange({ ...value, classifier_llm_config })} + />
Classifier Prompt diff --git a/ui/litellm-dashboard/src/components/add_model/ClassifierVisionConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassifierVisionConfig.tsx new file mode 100644 index 00000000000..34c41fff006 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/ClassifierVisionConfig.tsx @@ -0,0 +1,79 @@ +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import React from "react"; + +import type { ClassifierLLMConfigWire } from "./build_complexity_router_config"; + +export const DEFAULT_CLASSIFIER_VISION_ENABLED = false; +export const DEFAULT_CLASSIFIER_VISION_MAX_IMAGES = 1; + +const MAX_IMAGES_ID = "classifier-vision-max-images"; + +interface ClassifierVisionConfigProps { + value: ClassifierLLMConfigWire; + onChange: (value: ClassifierLLMConfigWire) => void; +} + +const ClassifierVisionConfig: React.FC = ({ value, onChange }) => { + const [draftMaxImages, setDraftMaxImages] = React.useState(null); + const enabled = value.vision?.enabled ?? DEFAULT_CLASSIFIER_VISION_ENABLED; + + const handleMaxImagesChange = (raw: string): void => { + setDraftMaxImages(raw); + const parsed = Number(raw); + if (raw.trim() === "" || !Number.isFinite(parsed)) return; + onChange({ + ...value, + vision: { ...value.vision, enabled, max_images: Math.max(1, Math.round(parsed)) }, + }); + }; + + return ( +
+
+ { + if (!visionEnabled) { + const { vision: _vision, ...withoutVision } = value; + onChange(withoutVision); + return; + } + onChange({ + ...value, + vision: { + ...value.vision, + enabled: true, + max_images: value.vision?.max_images ?? DEFAULT_CLASSIFIER_VISION_MAX_IMAGES, + }, + }); + }} + aria-label="Use images for classification" + /> + Use images for classification +
+ + Send inline image data to the classifier so it can choose a tier from what the image shows. + + {enabled && ( +
+ + handleMaxImagesChange(event.target.value)} + onBlur={() => setDraftMaxImages(null)} + className="w-full" + /> +
+ )} +
+ ); +}; + +export default ClassifierVisionConfig; diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 0590b524a06..2970e14b335 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -1,5 +1,6 @@ import { fireEvent, renderWithProviders, screen, within } from "../../../tests/test-utils"; import userEvent from "@testing-library/user-event"; +import React from "react"; import { vi } from "vitest"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; vi.mock( @@ -1690,3 +1691,83 @@ describe("ComplexityRouterConfig tier editing", () => { expect(screen.queryByText("Display names rename the built-in tiers", { exact: false })).not.toBeInTheDocument(); }); }); + +describe("classifier vision settings", () => { + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + + const VisionFixture = ({ onChange = vi.fn() }: { onChange?: ReturnType }) => { + const [value, setValue] = React.useState(llmValue); + return ( + { + setValue(nextValue); + onChange(nextValue); + }} + /> + ); + }; + + it("starts off and reveals the default cap when enabled", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + + const vision = screen.getByRole("switch", { name: "Use images for classification" }); + expect(vision).not.toBeChecked(); + expect(screen.queryByLabelText("Maximum images per request")).not.toBeInTheDocument(); + + fireEvent.click(vision); + + expect(screen.getByLabelText("Maximum images per request")).toHaveValue("1"); + }); + + it("writes the switch and a clamped image cap into the classifier config", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + + fireEvent.click(screen.getByRole("switch", { name: "Use images for classification" })); + expect(onChange).toHaveBeenLastCalledWith({ + ...llmValue, + classifier_llm_config: { ...llmValue.classifier_llm_config, vision: { enabled: true, max_images: 1 } }, + }); + + fireEvent.change(screen.getByLabelText("Maximum images per request"), { target: { value: "1.7" } }); + expect(onChange).toHaveBeenLastCalledWith({ + ...llmValue, + classifier_llm_config: { ...llmValue.classifier_llm_config, vision: { enabled: true, max_images: 2 } }, + }); + }); + + it("keeps the image cap draft empty until a valid value is entered", () => { + const onChange = vi.fn(); + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + fireEvent.click(screen.getByRole("switch", { name: "Use images for classification" })); + onChange.mockClear(); + + const input = screen.getByLabelText("Maximum images per request"); + fireEvent.change(input, { target: { value: "" } }); + + expect(input).toHaveValue(""); + expect(onChange).not.toHaveBeenCalled(); + + fireEvent.change(input, { target: { value: "0" } }); + expect(onChange).toHaveBeenLastCalledWith({ + ...llmValue, + classifier_llm_config: { ...llmValue.classifier_llm_config, vision: { enabled: true, max_images: 1 } }, + }); + }); + + it("is absent when the classifier is heuristic", () => { + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + + expect(screen.queryByText("Use images for classification")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 1b5bb9e72eb..17c1a83fd67 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -1170,12 +1170,13 @@ describe("buildComplexityRouterConfig stall escalation", () => { }); it("emits the toggle and both knobs when it is on", () => { - const config = buildComplexityRouterConfig({ + const params = { ...baseParams, stallEscalationEnabled: true, stallEscalationWindow: 8, stallEscalationRepeatThreshold: 4, - }); + }; + const config = buildComplexityRouterConfig(params); expect(config.stall_escalation_enabled).toBe(true); expect(config.stall_escalation_window).toBe(8); expect(config.stall_escalation_repeat_threshold).toBe(4); @@ -1207,3 +1208,38 @@ describe("dryRunRejection", () => { expect(dryRunRejection({ valid: true, error: null })).toBeNull(); }); }); + +describe("classifier vision wire payload", () => { + const vision = { enabled: true, max_images: 3 }; + const classifierLlmConfig = { model: "classifier", timeout_ms: 3000, vision }; + + it("keeps vision through the standard-tier payload", () => { + const params = { ...baseParams, classifierType: "llm" as const, classifierLlmConfig }; + const payload = buildComplexityRouterConfig(params); + + expect(payload.classifier_llm_config).toMatchObject({ vision }); + }); + + it("keeps vision through the custom-tier payload", () => { + const customTierSet = { + tiers: [ + { id: "simple", name: "simple", definition: "small talk", models: ["gpt-4o-mini"] }, + { id: "complex", name: "complex", definition: "hard work", models: ["gpt-4o"] }, + ], + fallback_tier_id: "simple", + }; + const payload = buildComplexityRouterConfig({ ...baseParams, customTierSet, classifierLlmConfig }); + + expect(payload.classifier_llm_config).toMatchObject({ vision }); + }); + + it("keeps an untouched classifier config free of vision", () => { + const payload = buildComplexityRouterConfig({ + ...baseParams, + classifierType: "llm", + classifierLlmConfig: { model: "classifier", timeout_ms: 3000 }, + }); + + expect(payload.classifier_llm_config).not.toHaveProperty("vision"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index d7974484970..7769fb832fe 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -1,8 +1,5 @@ -import { KeywordTierRule } from "./KeywordTierRules"; - -type ClassifierLLMConfigWire = ClassifierLLMConfig & { vision?: { enabled?: boolean; max_images?: number } }; - import type { ModelGroup } from "../llm_calls/fetch_models"; +import { KeywordTierRule } from "./KeywordTierRules"; import { type CustomTierSet, type TierRow, @@ -42,6 +39,9 @@ import { usesLlmClassifier, } from "./ComplexityRouterConfig"; +export type ClassifierVisionConfig = { enabled?: boolean; max_images?: number }; +export type ClassifierLLMConfigWire = ClassifierLLMConfig & { vision?: ClassifierVisionConfig }; + /** * Drop an empty system_prompt so the payload carries an override only when there is one. The * backend rejects a blank string rather than reading it as "use the default", and sending `""` @@ -124,7 +124,7 @@ export interface BuildComplexityRouterConfigParams { planModeMinTier: string | undefined; tierLabels: ComplexityTierLabels | undefined; classifierType: ClassifierType; - classifierLlmConfig: ClassifierLLMConfig | undefined; + classifierLlmConfig: ClassifierLLMConfigWire | undefined; classifierContextWindowSize: number | undefined; classifierContextBudgetChars: number | undefined; classifierContextIncludeAssistantTurns: boolean | undefined; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx index 9a55d3c0703..e45f84fa646 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.test.tsx @@ -1036,3 +1036,60 @@ describe("EditAutoRouterModal with a stored custom tier set", () => { expect(savedConfig().tier_model_configs).toEqual(CUSTOM_STORED.tier_model_configs); }); }); + +describe("EditAutoRouterModal classifier vision", () => { + beforeEach(() => { + modelPatchUpdateCall.mockClear(); + }); + + const STORED_CONFIG = { + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["gpt-4o-mini"], COMPLEX: ["gpt-4o-mini"], REASONING: ["gpt-4o-mini"] }, + classifier_type: "llm", + classifier_llm_config: { + model: "gpt-4o-mini", + timeout_ms: 3000, + vision: { enabled: true, max_images: 2 }, + }, + }; + + const renderModal = () => + renderWithProviders( + , + ); + + it("hydrates and keeps a stored vision setting through an untouched save", async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(await screen.findByText("Advanced: Classification Method")); + expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeChecked(); + expect(screen.getByLabelText("Maximum images per request")).toHaveValue("2"); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classifier_llm_config).toMatchObject({ vision: { enabled: true, max_images: 2 } }); + }); + + it("removes vision when the operator turns it off", async () => { + const user = userEvent.setup(); + renderModal(); + + await user.click(await screen.findByText("Advanced: Classification Method")); + await user.click(screen.getByRole("switch", { name: "Use images for classification" })); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); + expect(savedConfig().classifier_llm_config).not.toHaveProperty("vision"); + }); +}); From 29ac88ebc6bb93bda02138699c5ebe08bdd4e8cc Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 4 Sep 2026 23:59:51 -0700 Subject: [PATCH 8/9] fix(batches): register ownership for every batch create path (#39810) * fix(batches): register ownership for every batch create path Since the team isolation change, the managed files hook decided whether a response came from a create by looking for the managed input file id on it, which only the unified input path sets. Batches created from a model-encoded input file id, a model param, or a raw provider id with ?provider= never got an ownership row, so they vanished from GET /v1/batches for the key that created them. The create endpoint now stamps a create marker on the response before the hooks run, and the hook keys ownership registration and the batch-created metric on that marker instead of on the input id format. * test(batches): assert ownership registration through the managed files hook The endpoint tests asserted the private create marker, which is wiring, not behaviour. They now run the create through the real managed files hook and assert the ownership row is written for the creating key on every create path, with the unified path driven by a genuine encoded input file id instead of patched decoders. --- .../proxy/hooks/managed_files.py | 8 +-- litellm/proxy/batches_endpoints/endpoints.py | 3 + .../openai_files_endpoints/common_utils.py | 2 + .../proxy/hooks/test_managed_files.py | 16 ++--- .../proxy/test_managed_files_hook.py | 52 +++++++++++++++ .../proxy/batches_endpoints/test_endpoints.py | 64 ++++++++++++++++++- 6 files changed, 129 insertions(+), 16 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index e11c6e70540..bc1eb6cebc2 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -46,6 +46,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.openai_files_endpoints.common_utils import ( + BATCH_CREATE_HIDDEN_PARAM, FILE_LIST_CONTINUATION_CHUNK_SIZE, MAX_FILE_LIST_LIMIT, _is_base64_encoded_unified_file_id, @@ -1321,7 +1322,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ## Check if unified_file_id is in the response unified_file_id = response._hidden_params.get("unified_file_id") # managed file id unified_batch_id = response._hidden_params.get("unified_batch_id") # managed batch id - is_batch_create: Final = unified_file_id is not None + is_batch_create: Final = response._hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True model_id = cast(Optional[str], response._hidden_params.get("model_id")) model_name = cast(Optional[str], response._hidden_params.get("model_name")) @@ -1410,10 +1411,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): litellm_parent_otel_span=user_api_key_dict.parent_otel_span, ) - # Only record batch creation metric on actual create (not retrieve/cancel). - # unified_file_id in _hidden_params is only set by the create_batch endpoint. - original_unified_file_id = response._hidden_params.get("unified_file_id") - if original_unified_file_id: + if is_batch_create: prom_logger = self._get_prometheus_logger() if prom_logger: batch_provider = "" diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index be889a22cae..5c4bacd757c 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -25,6 +25,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( get_custom_llm_provider_from_request_query, ) from litellm.proxy.openai_files_endpoints.common_utils import ( + BATCH_CREATE_HIDDEN_PARAM, _is_base64_encoded_unified_file_id, add_internal_model_credentials, apply_team_provider_credentials, @@ -347,6 +348,8 @@ async def create_batch( **_create_batch_data, ) + response._hidden_params[BATCH_CREATE_HIDDEN_PARAM] = True + ### CALL HOOKS ### - modify outgoing data response = await proxy_logging_obj.post_call_success_hook( data=data, user_api_key_dict=user_api_key_dict, response=response diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 992ed0d814d..15eeddbc489 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -37,6 +37,8 @@ MAX_FILE_LIST_LIMIT: Final = 10000 FILE_LIST_CONTINUATION_CHUNK_SIZE: Final = 500 +BATCH_CREATE_HIDDEN_PARAM: Final = "batch_create" + def validate_file_list_limit(limit: int | None) -> None: """Reject a ``limit`` outside the range OpenAI documents for GET /v1/files.""" diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 57394f1cebe..7e96c956664 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -10,6 +10,7 @@ from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFi from litellm.caching import DualCache from litellm.proxy._types import CallTypes from litellm.proxy.openai_files_endpoints.common_utils import ( + BATCH_CREATE_HIDDEN_PARAM, _is_base64_encoded_unified_file_id, encode_file_id_with_model, ) @@ -3185,7 +3186,7 @@ def _batch_response(batch_id, output_file_id=None, is_create=False): output_file_id=output_file_id, ) if is_create: - batch._hidden_params["unified_file_id"] = "unified-input-file-id" + batch._hidden_params[BATCH_CREATE_HIDDEN_PARAM] = True return batch @@ -3411,11 +3412,8 @@ async def test_provider_format_file_without_ownership_row_stays_accessible(): @pytest.mark.asyncio -async def test_post_call_batch_create_stores_ownership_row(): - """ - Batch creation (response hidden params carry the unified input file id) - must write an ownership row attributed to the creating key. - """ +@pytest.mark.parametrize("batch_id", [MODEL_ENCODED_BATCH_ID, RAW_PROVIDER_BATCH_ID]) +async def test_post_call_batch_create_stores_ownership_row(batch_id): from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() @@ -3432,13 +3430,11 @@ async def test_post_call_batch_create_stores_ownership_row(): user_api_key_dict=UserAPIKeyAuth( user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() ), - response=_batch_response(MODEL_ENCODED_BATCH_ID, is_create=True), + response=_batch_response(batch_id, is_create=True), ) upsert_call = prisma_client.db.litellm_managedobjecttable.upsert.await_args - assert upsert_call.kwargs["where"] == { - "unified_object_id": MODEL_ENCODED_BATCH_ID - } + assert upsert_call.kwargs["where"] == {"unified_object_id": batch_id} create_data = upsert_call.kwargs["data"]["create"] assert create_data["created_by"] == "user_a" assert create_data["team_id"] == "team_a" diff --git a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py index f3ad8a8592e..091b958d7c3 100644 --- a/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py +++ b/tests/test_litellm/enterprise/proxy/test_managed_files_hook.py @@ -15,6 +15,7 @@ from typing import Optional from unittest.mock import AsyncMock, MagicMock, patch from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth +from litellm.proxy.openai_files_endpoints.common_utils import BATCH_CREATE_HIDDEN_PARAM from litellm.types.llms.openai import FileListPage, OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -1540,6 +1541,11 @@ async def test_batch_create_hook_persists_creating_key_and_tags(): managed_files = _make_managed_files_instance() creator = UserAPIKeyAuth(api_key="sk-the-creator", user_id="alice", parent_otel_span=None) create_response = _make_batch_response(status="validating", output_file_id=None) + create_response._hidden_params = { + BATCH_CREATE_HIDDEN_PARAM: True, + "model_id": "model-deploy-xyz", + "model_name": "azure/gpt-4", + } await managed_files.async_post_call_success_hook( data={"litellm_metadata": {"tags": ["env:prod", "team:ml"], "user_api_key": creator.api_key}}, @@ -1554,6 +1560,52 @@ async def test_batch_create_hook_persists_creating_key_and_tags(): assert stored["user_api_key_dict"] is creator +@pytest.mark.asyncio +async def test_batch_create_hook_records_created_metric_once(): + managed_files = _make_managed_files_instance() + prometheus_logger = MagicMock() + managed_files._get_prometheus_logger = MagicMock(return_value=prometheus_logger) + create_response = _make_batch_response(status="validating", output_file_id=None) + create_response._hidden_params = { + BATCH_CREATE_HIDDEN_PARAM: True, + "model_id": "model-deploy-xyz", + "model_name": "azure/gpt-4", + } + + await managed_files.async_post_call_success_hook( + data={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-the-creator", user_id="alice", parent_otel_span=None), + response=create_response, + ) + + prometheus_logger.record_managed_batch_created.assert_called_once() + recorded = prometheus_logger.record_managed_batch_created.call_args.kwargs + assert recorded["model"] == "azure/gpt-4" + assert recorded["api_provider"] == "azure" + assert recorded["user"] == "alice" + + +@pytest.mark.asyncio +async def test_batch_retrieve_hook_does_not_record_created_metric(): + managed_files = _make_managed_files_instance() + prometheus_logger = MagicMock() + managed_files._get_prometheus_logger = MagicMock(return_value=prometheus_logger) + retrieve_response = _make_batch_response(status="in_progress", output_file_id=None) + retrieve_response._hidden_params = { + "unified_batch_id": "some-unified-batch-id", + "model_id": "model-deploy-xyz", + "model_name": "azure/gpt-4", + } + + await managed_files.async_post_call_success_hook( + data={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-the-poller", user_id="bob", parent_otel_span=None), + response=retrieve_response, + ) + + prometheus_logger.record_managed_batch_created.assert_not_called() + + @pytest.mark.asyncio async def test_batch_retrieve_hook_does_not_claim_attribution(): """A retrieve carries unified_batch_id but no unified_file_id, so it must not rewrite diff --git a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py index 2f6a5a3b0e0..a37c8ff2bb4 100644 --- a/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/batches_endpoints/test_endpoints.py @@ -29,6 +29,7 @@ added to this layer raises instead of silently passing - the inventory of seams cannot drift without a test failure. """ +import base64 import json from contextlib import ExitStack from dataclasses import dataclass @@ -36,7 +37,7 @@ from typing import Any, Dict, Optional from unittest.mock import AsyncMock, MagicMock, patch import pytest - +from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles import litellm import litellm.proxy.batches_endpoints.endpoints as endpoints @@ -989,6 +990,67 @@ async def test_create__uses_acreate_batch_route_type(harness, openai_env_creds): assert harness.pre_call.call_args.kwargs["route_type"] == "acreate_batch" +def install_managed_files_hook(harness: Harness) -> AsyncMock: + prisma_client = AsyncMock() + managed_files = _PROXY_LiteLLMManagedFiles(MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client) + harness.logging.post_call_success_hook = AsyncMock(side_effect=managed_files.async_post_call_success_hook) + harness.router.model_list = [] + return prisma_client + + +TEAM_A_KEY = UserAPIKeyAuth(api_key="sk-team-a", user_id="user_a", team_id="team_a") + + +def assert_ownership_registered_for_team_a(prisma_client: AsyncMock, batch_id: str) -> None: + upsert = prisma_client.db.litellm_managedobjecttable.upsert + upsert.assert_awaited_once() + assert upsert.await_args.kwargs["where"] == {"unified_object_id": batch_id} + created = upsert.await_args.kwargs["data"]["create"] + assert created["created_by"] == "user_a" + assert created["team_id"] == "team_a" + prisma_client.db.litellm_managedobjecttable.update_many.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body", + [ + {"input_file_id": AZURE_FILE_ID}, + {"input_file_id": "file-plain", "model": "vertex-model"}, + {"input_file_id": "file-plain"}, + ], + ids=["model_encoded_file_id", "model_param", "provider_fallback"], +) +async def test_create__registers_ownership_for_creator(harness, openai_env_creds, body): + set_body(harness, {**body, "endpoint": "/v1/chat/completions", "completion_window": "24h"}) + prisma_client = install_managed_files_hook(harness) + + resp = await call_create(harness, user=TEAM_A_KEY) + + assert_ownership_registered_for_team_a(prisma_client, resp.id) + + +@pytest.mark.asyncio +async def test_create__unified_file_id_registers_ownership_for_creator(harness): + unified_input_file_id = base64.urlsafe_b64encode( + b"litellm_proxy:application/octet-stream;unified_id,input-uuid;target_model_names,gpt-4o-mini" + ).decode() + set_body( + harness, + { + "input_file_id": unified_input_file_id, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + }, + ) + prisma_client = install_managed_files_hook(harness) + + resp = await call_create(harness, user=TEAM_A_KEY) + + assert harness.router_acreate.call_count == 1 + assert_ownership_registered_for_team_a(prisma_client, resp.id) + + @pytest.mark.asyncio async def test_create__metadata_sanitized_before_forwarding(harness, openai_env_creds): set_body( From aca1c54391ceef578c31603fcd7872b267b451ca Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 01:26:36 -0700 Subject: [PATCH 9/9] refactor(proxy): build the OpenAI websocket refusal frame from a TypedDict The two dict literals behind the refusal event counted against the LIT002 ceiling once the base branch used up its headroom, so the frame is now a ReadOnly TypedDict built in one shot. Importing Literal explicitly also makes the UP037 suppression on the Vertex discovery signature unnecessary, so it goes. --- .../llm_passthrough_endpoints.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index e92c949299c..32da2658b99 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -16,12 +16,13 @@ import re from collections.abc import AsyncGenerator, Callable, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, Protocol, cast +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket from fastapi.responses import StreamingResponse from starlette.websockets import WebSocketState +from typing_extensions import ReadOnly, TypedDict import litellm from litellm import get_llm_provider @@ -1775,7 +1776,7 @@ def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str] def get_vertex_pass_through_handler( - call_type: Literal["discovery", "aiplatform"], # noqa: UP037 # ruff reports quoted Literal values here + call_type: Literal["discovery", "aiplatform"], ) -> BaseVertexAIPassThroughHandler: if call_type == "discovery": return VertexAIDiscoveryPassThroughHandler() @@ -2352,6 +2353,16 @@ class _OpenAIWebsocketRefusal: message: str +class _OpenAIWebsocketErrorDetail(TypedDict): + type: ReadOnly[Literal["invalid_request_error"]] + message: ReadOnly[str] + + +class _OpenAIWebsocketErrorFrame(TypedDict): + type: ReadOnly[Literal["error"]] + error: ReadOnly[_OpenAIWebsocketErrorDetail] + + _OPENAI_WS_DISABLED_REFUSAL: Final = _OpenAIWebsocketRefusal( close_reason="OpenAI websocket passthrough is disabled", message=( @@ -2451,14 +2462,11 @@ async def openai_websocket_proxy_route( refusal: Final = await _openai_websocket_refusal(user_api_key_dict, general_settings, model_allowlists) if refusal is not None: await websocket.accept(subprotocol=negotiated_subprotocol) - await websocket.send_text( - json.dumps( - { - "type": "error", - "error": {"type": "invalid_request_error", "message": refusal.message}, - } - ) - ) + error_frame: Final[_OpenAIWebsocketErrorFrame] = { + "type": "error", + "error": {"type": "invalid_request_error", "message": refusal.message}, + } + await websocket.send_text(json.dumps(error_frame)) await websocket.close(code=1008, reason=refusal.close_reason) return