From 50c90281f4fda1b080c2c4be91d8065f14f6f973 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 2 Jul 2026 13:24:22 -0700 Subject: [PATCH 001/101] feat(mcp): add true_passthrough and oauth_delegate auth modes Introduce two first-class MCP server auth_type values that make LiteLLM's role in upstream authentication explicit, added alongside the existing delegate_auth_to_upstream / oauth_passthrough flags without changing their behavior. true_passthrough is a transparent proxy: LiteLLM performs no admission auth, requires no x-litellm-api-key, mints/stores/refreshes nothing, and forwards the client's Authorization to the upstream exactly as received. oauth_delegate keeps normal LiteLLM admission (x-litellm-api-key / SSO / JWT) and then forwards the client's separate upstream Authorization unchanged; the admission credential is never forwarded upstream. Both modes forward the caller's token via the existing extra_headers path and defer egress credential resolution to v1 (the v2 to_server_spec returns None for them). Upstream 401/403 responses are surfaced rather than swallowed so upstream OAuth challenges are preserved. Servers in either mode require per-user auth, so userless health checks are skipped. --- .../mcp_server/auth/user_api_key_auth_mcp.py | 33 + .../mcp_server/mcp_server_manager.py | 95 ++- .../outbound_credentials/adapter.py | 10 +- .../outbound_credentials/resolver.py | 23 +- .../proxy/_experimental/mcp_server/server.py | 8 + litellm/types/mcp.py | 4 + .../types/mcp_server/mcp_server_manager.py | 15 + .../auth/test_user_api_key_auth_mcp.py | 677 +++++++----------- .../outbound_credentials/test_adapter.py | 7 + .../outbound_credentials/test_resolver.py | 25 +- .../mcp_server/test_mcp_server_manager.py | 223 ++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 12 files changed, 676 insertions(+), 446 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index d2986a3cd82..9f28da19292 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -220,6 +220,12 @@ class MCPRequestHandler: # when EVERY target is auth_type=oauth2 with delegate_auth_to_upstream # set; fails closed otherwise. validated_user_api_key_auth = UserAPIKeyAuth() + elif MCPRequestHandler._target_servers_are_true_passthrough( + path=request_route, + mcp_servers=mcp_servers, + client_ip=IPAddressUtils.get_mcp_client_ip(request), + ): + validated_user_api_key_auth = UserAPIKeyAuth() elif oauth2_headers: # Authorization on a non-delegated server: the bearer must be a real # LiteLLM credential, so a failed validation is a genuine 401/403 and @@ -399,6 +405,33 @@ class MCPRequestHandler: return False return True + @staticmethod + def _target_servers_are_true_passthrough( + path: str, mcp_servers: Optional[list[str]], client_ip: Optional[str] + ) -> bool: + """ + True only when EVERY MCP server the request targets is ``auth_type == true_passthrough``. + Fails closed when any target does not opt in or cannot be resolved. + + Used by :meth:`process_mcp_request` to skip LiteLLM admission auth entirely: the gateway is a + transparent proxy and the caller's ``Authorization`` is an upstream token, never a LiteLLM key. + Mirrors :meth:`_target_servers_delegate_auth_to_upstream`; a mixed-target request keeps normal auth. + """ + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + + target_names = MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers) + if not target_names: + return False + + for name in target_names: + server = global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip) + if server is None or server.auth_type != MCPAuth.true_passthrough: + return False + return True + @staticmethod def _resolve_target_server_names(path: str, mcp_servers_header: Optional[List[str]]) -> List[str]: """ diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 7c88c903324..fd978e5bfc9 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -78,6 +78,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_ ) from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( AuthorizationCodeConfig, + PassthroughConfig, ServerSpec, TokenExchangeConfig, ) @@ -213,6 +214,13 @@ def _should_strip_caller_authorization( pass-through cold-start case (RFC 9728) the bearer in ``Authorization`` is the upstream OAuth token and must be forwarded, so we keep it. + - **oauth_delegate servers**: admission always runs and there is no + anonymous path, so the caller's separate ``Authorization`` is + forwarded only when a distinct ``x-litellm-api-key`` carried + admission. Without that header the ``Authorization`` *was* the + admission credential — a virtual key, an IdP JWT, or an SSO / OIDC / + session token whose ``api_key`` is ``None`` — and must never reach + the upstream, so it is stripped regardless of the ``api_key`` value. """ if mcp_server.auth_type == MCPAuth.oauth2_token_exchange: # OBO: the inbound Authorization is the subject token. It is exchanged at the IdP and only the @@ -226,11 +234,13 @@ def _should_strip_caller_authorization( # upstream — it would override another user's stored credential. Delegate and # pass-through return None from to_server_spec and keep forwarding the bearer. return True - if not mcp_server.is_oauth_passthrough: + if not (mcp_server.is_oauth_passthrough or mcp_server.is_oauth_delegate): return False normalized_raw_headers = {str(k).lower(): v for k, v in (raw_headers or {}).items() if isinstance(k, str)} has_explicit_litellm_admission_header = normalized_raw_headers.get("x-litellm-api-key") is not None + if mcp_server.is_oauth_delegate: + return not has_explicit_litellm_admission_header admission_consumed_authorization_as_litellm_key = ( user_api_key_auth is not None and bool(getattr(user_api_key_auth, "api_key", None)) @@ -323,6 +333,41 @@ async def _resolve_byok_mcp_auth_header( return mcp_auth_header +def _client_forwarded_authorization_headers( + mcp_server: MCPServer, + oauth2_headers: Optional[dict[str, str]], + raw_headers: Optional[dict[str, str]], + user_api_key_auth: Optional[UserAPIKeyAuth], +) -> Optional[dict[str, str]]: + """Egress headers for the client-forwarded-token modes (``true_passthrough`` / ``oauth_delegate``). + + Forwards the caller's ``Authorization`` to the upstream, stripped when + ``_should_strip_caller_authorization`` says it was consumed as the LiteLLM admission key. Shared by + ``_call_regular_mcp_tool`` and ``server.py``'s ``_prepare_mcp_server_headers`` so the two egress + paths cannot drift, mirroring the ``_should_strip_caller_authorization`` split. + """ + extra_headers = oauth2_headers.copy() if oauth2_headers else None + if extra_headers and _should_strip_caller_authorization( + mcp_server=mcp_server, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ): + return _without_authorization(extra_headers) + return extra_headers + + +def _take_forwarded_authorization( + headers: Optional[dict[str, str]], +) -> tuple[Optional[str], Optional[dict[str, str]]]: + """Pop the ``Authorization`` value out of ``headers`` (case-insensitive), returning it with the + remaining headers, so the passthrough resolver arm is the single Authorization source rather than + the header also riding in ``extra_headers`` (which the resolved auth would then defer to).""" + if not headers: + return None, headers + value = next((v for k, v in headers.items() if k.lower() == "authorization"), None) + return value, _without_authorization(headers) + + def _extract_upstream_auth_failure( exc: BaseException, ) -> Optional[tuple[int, Optional[str]]]: @@ -1623,15 +1668,18 @@ class MCPServerManager: delegate_server_ids = [ server.server_id for server in self.get_registry().values() - if getattr(server, "auth_type", None) == MCPAuth.oauth2 - and getattr(server, "delegate_auth_to_upstream", False) is True - # M2M servers must not be exposed anonymously: an - # unauthenticated caller would get LiteLLM to proxy tool - # calls using its stored client_credentials. Resolve the flow - # rather than reading has_client_credentials so an unstamped - # M2M-shape row (null column, verbatim-read as non-M2M) still - # fails closed here, matching the anonymous-delegate auth gate. - and MCPServerManager.effective_oauth2_flow(server) != "client_credentials" + if ( + getattr(server, "auth_type", None) == MCPAuth.oauth2 + and getattr(server, "delegate_auth_to_upstream", False) is True + # M2M servers must not be exposed anonymously: an + # unauthenticated caller would get LiteLLM to proxy tool + # calls using its stored client_credentials. Resolve the flow + # rather than reading has_client_credentials so an unstamped + # M2M-shape row (null column, verbatim-read as non-M2M) still + # fails closed here, matching the anonymous-delegate auth gate. + and MCPServerManager.effective_oauth2_flow(server) != "client_credentials" + ) + or getattr(server, "auth_type", None) == MCPAuth.true_passthrough ] combined_servers.update(delegate_server_ids) @@ -2229,16 +2277,17 @@ class MCPServerManager: spec = None if transport == MCPTransport.stdio else to_server_spec(server) provider = cred_provider or self._cred_provider # A caller-supplied per-request override (mcp_auth_header / x-mcp-*) defers to the v1 path - # so it wins - except for the per-user modes the v2 resolver owns (authorization_code's - # stored token and token_exchange's RFC 8693 minted token). A caller must not be able to - # substitute another user's stored credential, nor silently disable the OBO exchange and - # forward an arbitrary bearer upstream, so we keep the v2 spec and ignore the override for - # both; the REST tools preview supplies its not-yet-persisted token through the resolver - # (cred_provider), never this path. + # so it wins - except for the modes the v2 resolver owns per-caller (authorization_code's + # stored token, token_exchange's RFC 8693 minted token, and the passthrough modes' + # forwarded caller token). A caller must not be able to substitute another user's stored + # credential, nor silently disable the OBO exchange and forward an arbitrary bearer + # upstream, so we keep the v2 spec and ignore the override for these; the REST tools + # preview supplies its not-yet-persisted token through the resolver (cred_provider), + # never this path. if ( spec is not None and mcp_auth_header - and not isinstance(spec.config, (AuthorizationCodeConfig, TokenExchangeConfig)) + and not isinstance(spec.config, (AuthorizationCodeConfig, PassthroughConfig, TokenExchangeConfig)) ): spec = None auth_value = ( @@ -2305,11 +2354,14 @@ class MCPServerManager: server_url = server.url or "" if spec is not None: + inbound_token = subject_token + if isinstance(spec.config, PassthroughConfig): + inbound_token, extra_headers = _take_forwarded_authorization(extra_headers) resolved_auth, extra_headers = await self._resolve_v2_auth( server=server, spec=spec, provider=provider, - subject_token=subject_token, + subject_token=inbound_token, user_api_key_auth=user_api_key_auth, extra_headers=extra_headers, ) @@ -3730,6 +3782,13 @@ class MCPServerManager: user_api_key_auth=user_api_key_auth, ): extra_headers = _without_authorization(extra_headers) + elif mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate: + extra_headers = _client_forwarded_authorization_headers( + mcp_server=mcp_server, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) if mcp_server.extra_headers and raw_headers: if extra_headers is None: diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py index 05896bfff74..e87e8081ced 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py @@ -23,6 +23,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( AuthorizationCodeConfig, CredError, NoneConfig, + PassthroughConfig, ServerSpec, SharedKey, Subject, @@ -62,9 +63,10 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]: an ``assert_never`` tail, so a newly added auth mode fails the type gate here until it is explicitly mapped or explicitly deferred, rather than silently falling through to v1. Live modes: ``none``, the static-header family (``api_key`` plus the Authorization schemes, - all shared-key), ``oauth2`` per-user tokens (``authorization_code``), and - ``oauth2_token_exchange`` (OBO); client_credentials (M2M), delegated/passthrough - oauth2, and SigV4 return None and stay on v1. + all shared-key), ``oauth2`` per-user tokens (``authorization_code``), ``oauth2_token_exchange`` + (OBO), and the client-forwarded token modes ``true_passthrough`` / ``oauth_delegate`` + (``PassthroughConfig``); client_credentials (M2M), delegated/passthrough oauth2, and SigV4 + return None and stay on v1. """ if server.is_byok: return None # per-user BYOK source not migrated yet -> defer to v1 (any auth_type) @@ -94,6 +96,8 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]: ) # client_credentials (M2M) and delegate/passthrough oauth2 stay on v1 return None + case MCPAuth.true_passthrough | MCPAuth.oauth_delegate: + return ServerSpec(server_id=server.server_id, resource=resource, config=PassthroughConfig()) case MCPAuth.oauth2_token_exchange: return _token_exchange_spec(server, resource) case MCPAuth.aws_sigv4: diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py index c82ce1037d6..ecfd471190c 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py @@ -7,10 +7,11 @@ no precedence cascade. It is wildcard-free with an `assert_never` tail, so addin an arm fails the type gate (basedpyright `reportMatchNotExhaustive`); a bypassed gate fails loudly at runtime instead of returning `None`. -`none` and `api_key` (shared-key source) are live, as is `authorization_code`, which reads the -user's token from the injected `OAuthTokenStore`, and `token_exchange`, which swaps the caller's -inbound token through the injected `TokenExchanger`. The remaining arms are `not_implemented` stubs -that each land in a follow-up PR with their seam. Pure v2: no imports from v1. +`none`, `api_key` (shared-key source), and `passthrough` (forwards the caller's own inbound token) +are live, as is `authorization_code`, which reads the user's token from the injected +`OAuthTokenStore`, and `token_exchange`, which swaps the caller's inbound token through the +injected `TokenExchanger`. The remaining arms are `not_implemented` stubs that each land in a +follow-up PR with their seam. Pure v2: no imports from v1. """ from __future__ import annotations @@ -97,7 +98,7 @@ class UpstreamCredentialProvider: case ApiKeyConfig() as config: return self._api_key(config) case PassthroughConfig(): - return _not_implemented(AuthSpecKind.passthrough) + return self._passthrough(subject) case ClientCredentialsConfig(): return _not_implemented(AuthSpecKind.client_credentials) case TokenExchangeConfig() as config: @@ -118,6 +119,18 @@ class UpstreamCredentialProvider: """ return await self._authz_token(subject, server) is not None + def _passthrough(self, subject: Subject) -> Result[httpx.Auth, CredError]: + """Forward the caller's own upstream credential verbatim; the gateway mints nothing. + + The inbound token is the caller's already-disambiguated ``Authorization`` (never the LiteLLM + admission credential; the edge adapter drops that before building the ``Subject``). When it is + absent the request is sent unauthenticated so the upstream's own 401 surfaces, rather than the + gateway challenging on the upstream's behalf. + """ + if subject.inbound_token is None: + return Ok(NoOpAuth()) + return Ok(StaticHeaderAuth(subject.inbound_token.get_secret_value(), header_name="Authorization")) + def _api_key(self, config: ApiKeyConfig) -> Result[httpx.Auth, CredError]: match config.key_source: case SharedKey() as source: diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index e3812522ded..96fd014fbe1 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -331,6 +331,7 @@ if MCP_AVAILABLE: ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, + _client_forwarded_authorization_headers, _should_strip_caller_authorization, _without_authorization, global_mcp_server_manager, @@ -1561,6 +1562,13 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, ): extra_headers = _without_authorization(extra_headers) + elif server.is_true_passthrough or server.is_oauth_delegate: + extra_headers = _client_forwarded_authorization_headers( + mcp_server=server, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) if server.extra_headers and raw_headers: if extra_headers is None: diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index 9c564a3c7a6..d273e8ec4db 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -38,6 +38,8 @@ class MCPAuth(str, enum.Enum): aws_sigv4 = "aws_sigv4" token = "token" oauth2_token_exchange = "oauth2_token_exchange" + true_passthrough = "true_passthrough" + oauth_delegate = "oauth_delegate" # RFC 8693 default subject_token_type. A NULL column / omitted config key means @@ -60,6 +62,8 @@ MCPAuthType = Optional[ MCPAuth.aws_sigv4, MCPAuth.token, MCPAuth.oauth2_token_exchange, + MCPAuth.true_passthrough, + MCPAuth.oauth_delegate, ] ] diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 522a6f09165..f102ab5b7b9 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -152,6 +152,18 @@ class MCPServer(BaseModel): """True if this is an OAuth2 server that relies on per-user tokens (no client_credentials).""" return self.auth_type == MCPAuth.oauth2 and not self.has_client_credentials + @property + def is_true_passthrough(self) -> bool: + """True for the transparent-proxy mode: LiteLLM performs no admission auth and forwards the + client's ``Authorization`` to the upstream unchanged.""" + return self.auth_type == MCPAuth.true_passthrough + + @property + def is_oauth_delegate(self) -> bool: + """True for the delegated-upstream-OAuth mode: LiteLLM still admits the caller (API key / SSO / + JWT) but forwards the caller's separate upstream ``Authorization`` unchanged, minting nothing.""" + return self.auth_type == MCPAuth.oauth_delegate + @property def requires_per_user_auth(self) -> bool: """ @@ -167,6 +179,9 @@ class MCPServer(BaseModel): if self.needs_user_oauth_token: return True + if self.is_true_passthrough or self.is_oauth_delegate: + return True + # PAT passthrough: auth_type is none but extra_headers includes auth headers if self.auth_type == MCPAuth.none and self.extra_headers: auth_header_names = {"authorization", "x-api-key", "api-key", "apikey"} diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 3db0f8540f9..c984ccb783e 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -1,14 +1,12 @@ import json import os import sys -from unittest.mock import AsyncMock, MagicMock, call as mock_call, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient -sys.path.insert( - 0, os.path.abspath("../../../..") -) # Adds the parent directory to the system path +sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path from starlette.datastructures import Headers @@ -78,20 +76,14 @@ class TestMCPRequestHandler: ) # Mock the helper methods instead of database calls - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_key" - ) as mock_key_servers: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_team" - ) as mock_team_servers: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key") as mock_key_servers: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team") as mock_team_servers: # Set up return values mock_key_servers.return_value = key_servers mock_team_servers.return_value = team_servers # Call the method - result = await MCPRequestHandler.get_allowed_mcp_servers( - user_api_key_auth=mock_user_auth - ) + result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=mock_user_auth) # Assert the result (order-independent comparison) assert sorted(result) == sorted(expected_result) @@ -148,20 +140,14 @@ class TestMCPRequestHandler: ) # Mock the helper functions - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_key" - ) as mock_key_servers: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_team" - ) as mock_team_servers: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key") as mock_key_servers: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team") as mock_team_servers: # Configure mocks to return the test data mock_key_servers.return_value = key_servers mock_team_servers.return_value = team_servers # Call the method - result = await MCPRequestHandler.get_allowed_mcp_servers( - user_api_key_auth - ) + result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth) # Assert the result (order-independent comparison) assert sorted(result) == sorted(expected_servers) @@ -186,9 +172,7 @@ class TestMCPRequestHandler: ): """The require_key_mcp_access_defined general setting flips an empty key from inheriting its team's MCP servers (default) to inheriting none.""" - auth = UserAPIKeyAuth( - api_key="test-key", user_id="test-user", team_id="test-team" - ) + auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user", team_id="test-team") with ( patch.object( MCPRequestHandler, @@ -272,23 +256,22 @@ class TestMCPRequestHandler: async def test_no_mcp_servers_sentinel_returns_empty(self, team_servers): """A key scoped to the no-mcp-servers sentinel resolves to zero servers, overriding team inheritance and never leaking the sentinel marker.""" - user_api_key_auth = UserAPIKeyAuth( - api_key="test-key", user_id="test-user", team_id="test-team" - ) + user_api_key_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user", team_id="test-team") key_object_permission = MagicMock() - key_object_permission.mcp_servers = [ - SpecialMCPServerNames.no_mcp_servers.value - ] + key_object_permission.mcp_servers = [SpecialMCPServerNames.no_mcp_servers.value] - with patch.object( - MCPRequestHandler, - "_get_key_object_permission", - return_value=key_object_permission, - ), patch.object( - MCPRequestHandler, - "_get_allowed_mcp_servers_for_team", - new_callable=AsyncMock, - return_value=team_servers, + with ( + patch.object( + MCPRequestHandler, + "_get_key_object_permission", + return_value=key_object_permission, + ), + patch.object( + MCPRequestHandler, + "_get_allowed_mcp_servers_for_team", + new_callable=AsyncMock, + return_value=team_servers, + ), ): result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth) @@ -309,9 +292,7 @@ class TestMCPRequestHandler: "_get_key_object_permission", return_value=key_object_permission, ): - result = await MCPRequestHandler._get_allowed_mcp_servers_for_key( - user_api_key_auth - ) + result = await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_auth) assert result == [SpecialMCPServerNames.no_mcp_servers.value] @@ -320,9 +301,7 @@ class TestMCPRequestHandler: # Test case: None values in database mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_objectpermissiontable.find_unique.return_value = ( - None - ) + mock_prisma_client.db.litellm_objectpermissiontable.find_unique.return_value = None mock_prisma_client.db.litellm_teamtable.find_unique.return_value = None user_api_key_auth = UserAPIKeyAuth( @@ -337,9 +316,7 @@ class TestMCPRequestHandler: assert result == [] # Test case: Exception handling - mock_prisma_client.db.litellm_objectpermissiontable.find_unique.side_effect = ( - Exception("DB Error") - ) + mock_prisma_client.db.litellm_objectpermissiontable.find_unique.side_effect = Exception("DB Error") with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client): result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth) @@ -384,15 +361,9 @@ class TestMCPRequestHandler: access_group_ids=["grp-mcp"], ) with ( - patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_key" - ) as mock_key, - patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_team" - ) as mock_team, - patch.object( - MCPRequestHandler, "_get_key_access_group_mcp_server_extras" - ) as mock_grants, + patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key") as mock_key, + patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team") as mock_team, + patch.object(MCPRequestHandler, "_get_key_access_group_mcp_server_extras") as mock_grants, ): mock_key.return_value = key_servers mock_team.return_value = team_servers @@ -414,13 +385,9 @@ class TestMCPRequestHandler: "litellm.proxy.auth.auth_checks._get_mcp_server_ids_from_access_groups", new=AsyncMock(return_value=[]), ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - auth - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(auth) assert result == [] # expand_permission_list must not be reached when there are no raw ids. mock_mgr.expand_permission_list.assert_not_called() @@ -433,14 +400,10 @@ class TestMCPRequestHandler: "litellm.proxy.auth.auth_checks._get_mcp_server_ids_from_access_groups", new=AsyncMock(return_value=["alias-a", "srv-b"]), ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.expand_permission_list.return_value = ["srv-a", "srv-b"] - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - auth - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(auth) assert sorted(result) == ["srv-a", "srv-b"] mock_mgr.expand_permission_list.assert_called_once_with(["alias-a", "srv-b"]) @@ -451,9 +414,7 @@ class TestMCPRequestHandler: "litellm.proxy.auth.auth_checks._get_mcp_server_ids_from_access_groups", new=AsyncMock(side_effect=Exception("db down")), ): - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - auth - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(auth) assert result == [] @pytest.mark.parametrize( @@ -743,9 +704,7 @@ class TestMCPRequestHandler: # Verify MCP servers mcp_servers_header = extracted_headers.get(SpecialHeaders.mcp_servers.value) mcp_servers = None - if ( - mcp_servers_header is not None - ): # Changed from 'if mcp_servers_header:' to handle empty strings + if mcp_servers_header is not None: # Changed from 'if mcp_servers_header:' to handle empty strings try: # First try to parse as JSON array for backward compatibility try: @@ -754,16 +713,12 @@ class TestMCPRequestHandler: mcp_servers = None except (json.JSONDecodeError, TypeError, ValueError): # If JSON parsing fails, treat as comma-separated list - mcp_servers = [ - s.strip() for s in mcp_servers_header.split(",") if s.strip() - ] + mcp_servers = [s.strip() for s in mcp_servers_header.split(",") if s.strip()] except Exception: mcp_servers = None # If we got an empty string or parsing resulted in no servers, return empty list - if mcp_servers_header == "" or ( - mcp_servers is not None and len(mcp_servers) == 0 - ): + if mcp_servers_header == "" or (mcp_servers is not None and len(mcp_servers) == 0): mcp_servers = [] assert mcp_servers == expected_result["mcp_servers"] @@ -833,9 +788,7 @@ class TestMCPOAuth2AuthFlow: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = oauth2_server ( @@ -851,10 +804,7 @@ class TestMCPOAuth2AuthFlow: # The upstream token is never validated as a LiteLLM key ... mock_auth.assert_not_called() # ... and is preserved for upstream forwarding. - assert ( - oauth2_headers.get("Authorization") - == "Bearer atlassian-oauth2-access-token-xyz" - ) + assert oauth2_headers.get("Authorization") == "Bearer atlassian-oauth2-access-token-xyz" async def test_explicit_litellm_key_with_oauth2_authorization(self): """ @@ -893,9 +843,7 @@ class TestMCPOAuth2AuthFlow: assert call_args.kwargs["api_key"] == "sk-litellm-valid-key" # OAuth2 headers should still contain the Authorization token - assert ( - oauth2_headers.get("Authorization") == "Bearer atlassian-oauth2-token" - ) + assert oauth2_headers.get("Authorization") == "Bearer atlassian-oauth2-token" async def test_litellm_key_in_authorization_backward_compat(self): """ @@ -997,9 +945,7 @@ class TestMCPOAuth2AuthFlow: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_proxy_exception, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = oauth2_server with pytest.raises(ProxyException) as exc_info: @@ -1068,9 +1014,7 @@ class TestMCPPublicRouteGuard: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): # Explicit unresolvable target — proves auth still fails even # when the registry has no info to fall back to. @@ -1101,9 +1045,7 @@ class TestMCPPublicRouteGuard: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -1157,9 +1099,7 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp._is_mcp_passthrough_cold_start" ) as mock_cold_start, @@ -1199,9 +1139,7 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -1229,9 +1167,7 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -1263,18 +1199,14 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.IPAddressUtils.get_mcp_client_ip", return_value="203.0.113.10", ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(scope) assert exc_info.value.status_code == 401 - mock_mgr.get_mcp_server_by_name.assert_any_call( - "passthrough_server", client_ip="203.0.113.10" - ) + mock_mgr.get_mcp_server_by_name.assert_any_call("passthrough_server", client_ip="203.0.113.10") async def test_cold_start_propagates_non_401_http_error(self): from fastapi import HTTPException @@ -1294,9 +1226,7 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_forbidden, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -1329,9 +1259,7 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_server_error, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -1357,9 +1285,7 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -1367,9 +1293,7 @@ class TestMCPPassthroughColdStartAdmission: auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) assert isinstance(auth_result, UserAPIKeyAuth) - mock_mgr.get_mcp_server_by_name.assert_any_call( - "passthrough_server", client_ip="" - ) + mock_mgr.get_mcp_server_by_name.assert_any_call("passthrough_server", client_ip="") async def test_cold_start_allows_proxy_exception_401_for_path_target(self): from litellm.proxy._types import ProxyException @@ -1394,9 +1318,7 @@ class TestMCPPassthroughColdStartAdmission: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = ( TestMCPPassthroughColdStartAdmission._make_passthrough_server() @@ -1404,9 +1326,7 @@ class TestMCPPassthroughColdStartAdmission: auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) assert isinstance(auth_result, UserAPIKeyAuth) - mock_mgr.get_mcp_server_by_name.assert_any_call( - "passthrough_server", client_ip="" - ) + mock_mgr.get_mcp_server_by_name.assert_any_call("passthrough_server", client_ip="") @pytest.mark.asyncio @@ -1450,12 +1370,10 @@ class TestMCPOAuth2FallbackTargetGating: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPOAuth2FallbackTargetGating._make_server(MCPAuth.api_key) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPOAuth2FallbackTargetGating._make_server( + MCPAuth.api_key ) with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(scope) @@ -1483,9 +1401,7 @@ class TestMCPOAuth2FallbackTargetGating: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -1523,12 +1439,10 @@ class TestMCPOAuth2FallbackTargetGating: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPOAuth2FallbackTargetGating._make_server(MCPAuth.oauth2) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPOAuth2FallbackTargetGating._make_server( + MCPAuth.oauth2 ) with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(scope) @@ -1562,15 +1476,11 @@ class TestMCPOAuth2FallbackTargetGating: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPOAuth2FallbackTargetGating._make_server( - auth_type=MCPAuth.none, - is_oauth_passthrough=True, - ) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPOAuth2FallbackTargetGating._make_server( + auth_type=MCPAuth.none, + is_oauth_passthrough=True, ) auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) assert isinstance(auth_result, UserAPIKeyAuth) @@ -1598,9 +1508,7 @@ class TestMCPOAuth2FallbackTargetGating: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.IPAddressUtils.get_mcp_client_ip", return_value="203.0.113.10", ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -1612,9 +1520,7 @@ class TestMCPOAuth2FallbackTargetGating: # resolve to ``None`` (hidden by client IP) so neither bypass # opens. Use ``assert_any_call`` to assert the IP-scoped lookup # happened without locking the count. - mock_mgr.get_mcp_server_by_name.assert_any_call( - "hidden_oauth2_server", client_ip="203.0.113.10" - ) + mock_mgr.get_mcp_server_by_name.assert_any_call("hidden_oauth2_server", client_ip="203.0.113.10") async def test_fallback_blocked_when_any_target_in_header_is_not_oauth2(self): """ @@ -1649,9 +1555,7 @@ class TestMCPOAuth2FallbackTargetGating: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = mock_lookup with pytest.raises(HTTPException) as exc_info: @@ -1732,17 +1636,10 @@ class TestMCPDelegateAuthToUpstream: delegate_auth_to_upstream=True, available_on_public_internet=True, ) - assert ( - manager._build_mcp_server_table(delegated).delegate_auth_to_upstream is True - ) + assert manager._build_mcp_server_table(delegated).delegate_auth_to_upstream is True - not_delegated = delegated.model_copy( - update={"delegate_auth_to_upstream": False} - ) - assert ( - manager._build_mcp_server_table(not_delegated).delegate_auth_to_upstream - is False - ) + not_delegated = delegated.model_copy(update={"delegate_auth_to_upstream": False}) + assert manager._build_mcp_server_table(not_delegated).delegate_auth_to_upstream is False def test_build_mcp_server_table_preserves_oauth_passthrough(self): """Registry → API list rows must expose oauth_passthrough for the UI. @@ -1773,9 +1670,7 @@ class TestMCPDelegateAuthToUpstream: assert row.delegate_auth_to_upstream is False not_passthrough = passthrough.model_copy(update={"oauth_passthrough": False}) - assert ( - manager._build_mcp_server_table(not_passthrough).oauth_passthrough is False - ) + assert manager._build_mcp_server_table(not_passthrough).oauth_passthrough is False async def test_delegate_skips_litellm_auth_with_no_authorization(self): """ @@ -1796,15 +1691,11 @@ class TestMCPDelegateAuthToUpstream: patch( "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPDelegateAuthToUpstream._make_server( - auth_type=MCPAuth.oauth2, - delegate_auth_to_upstream=True, - ) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, ) auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) assert isinstance(auth_result, UserAPIKeyAuth) @@ -1834,15 +1725,11 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", new_callable=AsyncMock, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPDelegateAuthToUpstream._make_server( - auth_type=MCPAuth.oauth2, - delegate_auth_to_upstream=True, - ) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, ) ( auth_result, @@ -1880,15 +1767,11 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPDelegateAuthToUpstream._make_server( - auth_type=MCPAuth.oauth2, - delegate_auth_to_upstream=False, - ) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=False, ) with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(scope) @@ -1919,15 +1802,11 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPDelegateAuthToUpstream._make_server( - auth_type=MCPAuth.api_key, - delegate_auth_to_upstream=True, - ) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.api_key, + delegate_auth_to_upstream=True, ) with pytest.raises(HTTPException) as exc_info: await MCPRequestHandler.process_mcp_request(scope) @@ -1971,9 +1850,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = mock_lookup with pytest.raises(HTTPException) as exc_info: @@ -2003,9 +1880,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_user_api_key_auth_fails, ), - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = None with pytest.raises(HTTPException) as exc_info: @@ -2034,15 +1909,11 @@ class TestMCPDelegateAuthToUpstream: new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id="real-user"), ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPDelegateAuthToUpstream._make_server( - auth_type=MCPAuth.oauth2, - delegate_auth_to_upstream=True, - ) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, ) auth_result, *_rest = await MCPRequestHandler.process_mcp_request(scope) assert isinstance(auth_result, UserAPIKeyAuth) @@ -2074,15 +1945,11 @@ class TestMCPDelegateAuthToUpstream: new_callable=AsyncMock, return_value=UserAPIKeyAuth(user_id="real-user"), ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): - mock_mgr.get_mcp_server_by_name.return_value = ( - TestMCPDelegateAuthToUpstream._make_server( - auth_type=MCPAuth.oauth2, - delegate_auth_to_upstream=True, - ) + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, ) ( auth_result, @@ -2135,9 +2002,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = m2m_server # No delegate bypass → normal auth is attempted → 401 raised @@ -2188,9 +2053,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = legacy_m2m_server with pytest.raises(HTTPException) as exc_info: @@ -2235,9 +2098,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = pkce_server auth, *_rest = await MCPRequestHandler.process_mcp_request(scope) @@ -2278,9 +2139,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.return_value = internal_server auth, *_rest = await MCPRequestHandler.process_mcp_request(scope) @@ -2426,6 +2285,104 @@ class TestMCPDelegateAuthToUpstream: assert "public-server" in result assert "internal-server" in result + async def test_true_passthrough_skips_litellm_auth_anonymously(self): + """auth_type=true_passthrough performs no admission auth: the caller's Authorization is an + upstream token forwarded unchanged and user_api_key_auth is never called.""" + from litellm.types.mcp import MCPAuth + + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/true_passthrough_server", + "headers": [(b"authorization", b"Bearer upstream-token")], + } + + with ( + patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + new_callable=AsyncMock, + ) as mock_auth, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.return_value = TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.true_passthrough, + ) + ( + auth_result, + _, + _, + _, + oauth2_headers, + _, + ) = await MCPRequestHandler.process_mcp_request(scope) + assert isinstance(auth_result, UserAPIKeyAuth) + assert auth_result.api_key is None + assert oauth2_headers.get("Authorization") == "Bearer upstream-token" + mock_auth.assert_not_called() + + async def test_true_passthrough_mixed_targets_fail_closed(self): + """One true_passthrough target mixed with a non-passthrough target must NOT skip admission.""" + from fastapi import HTTPException + + from litellm.types.mcp import MCPAuth + + scope = { + "type": "http", + "method": "POST", + "path": "/mcp", + "headers": [(b"x-mcp-servers", b"tp_server,plain_server")], + } + + async def mock_user_api_key_auth_fails(api_key, request): + raise HTTPException(status_code=401, detail="Invalid API key") + + def mock_lookup(name, client_ip=None): + if name == "tp_server": + return TestMCPDelegateAuthToUpstream._make_server( + auth_type=MCPAuth.true_passthrough, + ) + return TestMCPDelegateAuthToUpstream._make_server(auth_type=MCPAuth.api_key) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", + side_effect=mock_user_api_key_auth_fails, + ), + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, + ): + mock_mgr.get_mcp_server_by_name.side_effect = mock_lookup + with pytest.raises(HTTPException) as exc_info: + await MCPRequestHandler.process_mcp_request(scope) + assert exc_info.value.status_code == 401 + + async def test_get_allowed_servers_includes_true_passthrough(self): + """Anonymous callers can reach true_passthrough servers; admission is delegated upstream.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + MCPServerManager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + manager = MCPServerManager() + tp_server = MCPServer( + server_id="tp-server", + name="tp_server", + transport="http", + auth_type=MCPAuth.true_passthrough, + available_on_public_internet=True, + ) + manager.registry = {tp_server.server_id: tp_server} + + with patch.object( + MCPRequestHandler, + "get_allowed_mcp_servers", + new_callable=AsyncMock, + return_value=[], + ): + result = await manager.get_allowed_mcp_servers(None) + + assert "tp-server" in result + def test_extract_target_server_names_matches_routing_parser(self): """ Regression: _extract_target_server_names_from_path must match the @@ -2464,13 +2421,12 @@ class TestMCPDelegateAuthToUpstream: ("/", []), ] for path_input, expected in cases: - assert ( - MCPRequestHandler._extract_target_server_names_from_path(path_input) - == expected - ), f"path={path_input!r} → expected {expected!r}" - assert ( - _get_mcp_servers_in_path(path_input) or [] - ) == expected, f"path={path_input!r} → routing expected {expected!r}" + assert MCPRequestHandler._extract_target_server_names_from_path(path_input) == expected, ( + f"path={path_input!r} → expected {expected!r}" + ) + assert (_get_mcp_servers_in_path(path_input) or []) == expected, ( + f"path={path_input!r} → routing expected {expected!r}" + ) async def test_delegate_does_not_bypass_on_extra_path_segment(self): """ @@ -2514,9 +2470,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = lookup_by_name with pytest.raises(HTTPException) as exc_info: @@ -2579,9 +2533,7 @@ class TestMCPDelegateAuthToUpstream: "litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp.user_api_key_auth", side_effect=mock_auth_raises, ) as mock_auth, - patch( - "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager" - ) as mock_mgr, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager") as mock_mgr, ): mock_mgr.get_mcp_server_by_name.side_effect = lookup_by_name # Bypass MUST NOT fire — path-derived target is the non-delegate @@ -2601,15 +2553,12 @@ class TestMCPDelegateAuthToUpstream: empty-list case, which fails closed). """ # Path matches /mcp/... — header is ignored. - assert MCPRequestHandler._resolve_target_server_names( - path="/mcp/foo", mcp_servers_header=["evil"] - ) == ["foo"] - assert MCPRequestHandler._resolve_target_server_names( - path="/mcp/foo,bar", mcp_servers_header=["evil"] - ) == ["foo", "bar"] - assert MCPRequestHandler._resolve_target_server_names( - path="/foo/mcp", mcp_servers_header=["evil"] - ) == ["foo"] + assert MCPRequestHandler._resolve_target_server_names(path="/mcp/foo", mcp_servers_header=["evil"]) == ["foo"] + assert MCPRequestHandler._resolve_target_server_names(path="/mcp/foo,bar", mcp_servers_header=["evil"]) == [ + "foo", + "bar", + ] + assert MCPRequestHandler._resolve_target_server_names(path="/foo/mcp", mcp_servers_header=["evil"]) == ["foo"] # Path does not match — header is trusted. assert MCPRequestHandler._resolve_target_server_names( path="/.well-known/oauth-authorization-server", @@ -2653,16 +2602,12 @@ class TestMCPCustomHeaderName: (None, "", "x-mcp-auth"), ], ) - def test_get_mcp_client_side_auth_header_name( - self, env_var, general_setting, expected_header_name - ): + def test_get_mcp_client_side_auth_header_name(self, env_var, general_setting, expected_header_name): """Test that custom header name configuration works correctly""" # Mock the secret manager and general settings with patch("litellm.secret_managers.main.get_secret_str") as mock_get_secret: - with patch( - "litellm.proxy.proxy_server.general_settings" - ) as mock_general_settings: + with patch("litellm.proxy.proxy_server.general_settings") as mock_general_settings: # Configure mocks mock_get_secret.return_value = env_var mock_general_settings.get.return_value = general_setting @@ -2685,9 +2630,7 @@ class TestMCPCustomHeaderName: if env_var is None: # When env var is None, general settings should be checked (twice if not None) expected_general_calls = 2 if general_setting is not None else 1 - assert ( - mock_general_settings.get.call_count == expected_general_calls - ) + assert mock_general_settings.get.call_count == expected_general_calls for call in mock_general_settings.get.call_args_list: assert call.args == ("mcp_client_side_auth_header_name",) else: @@ -2728,9 +2671,7 @@ class TestMCPCustomHeaderName: ), ], ) - def test_get_mcp_auth_header_from_headers_with_custom_name( - self, custom_header_name, headers, expected_auth_header - ): + def test_get_mcp_auth_header_from_headers_with_custom_name(self, custom_header_name, headers, expected_auth_header): """Test that MCP auth header extraction uses custom header name""" # Mock the header name method @@ -2749,9 +2690,7 @@ class TestMCPCustomHeaderName: extracted_headers = MCPRequestHandler._safe_get_headers_from_scope(scope) # Call the method - result = MCPRequestHandler._get_mcp_auth_header_from_headers( - extracted_headers - ) + result = MCPRequestHandler._get_mcp_auth_header_from_headers(extracted_headers) # Assert the result assert result == expected_auth_header @@ -2818,9 +2757,7 @@ class TestMCPCustomHeaderName: from starlette.datastructures import Headers # Test case 1: No server-specific headers - headers = Headers( - {"x-litellm-api-key": "test-key", "content-type": "application/json"} - ) + headers = Headers({"x-litellm-api-key": "test-key", "content-type": "application/json"}) result = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers) assert result == {} @@ -2904,17 +2841,13 @@ class TestMCPCustomHeaderName: assert result == {"github_mcp": {"Authorization": "Bearer github-mcp-token"}} # Test case 8: Edge case - empty header value - headers = Headers( - {"x-litellm-api-key": "test-key", "x-mcp-github-authorization": ""} - ) + headers = Headers({"x-litellm-api-key": "test-key", "x-mcp-github-authorization": ""}) result = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers) assert result == {"github": {"Authorization": ""}} # Test case 9: Edge case - very long header value long_token = "Bearer " + "x" * 1000 - headers = Headers( - {"x-litellm-api-key": "test-key", "x-mcp-github-authorization": long_token} - ) + headers = Headers({"x-litellm-api-key": "test-key", "x-mcp-github-authorization": long_token}) result = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers) assert result == {"github": {"Authorization": long_token}} @@ -2980,9 +2913,7 @@ class TestMCPAccessGroupsE2E: # Assert the results assert auth_result.api_key == "test-api-key" assert mcp_auth_header is None - assert ( - mcp_servers is None - ) # x-mcp-access-groups is not parsed as mcp_servers + assert mcp_servers is None # x-mcp-access-groups is not parsed as mcp_servers assert mcp_server_auth_headers == {} # Verify the mock was called @@ -3097,9 +3028,7 @@ def test_mcp_path_based_server_segregation(monkeypatch): # Use TestClient to make a request to /mcp/zapier,group1/tools client = TestClient(app) - response = client.get( - "/mcp/zapier,group1/tools", headers={"x-litellm-api-key": "test"} - ) + response = client.get("/mcp/zapier,group1/tools", headers={"x-litellm-api-key": "test"}) assert response.status_code == 200 assert response.json() == {"status": "ok"} @@ -3177,15 +3106,11 @@ async def test_get_team_object_permission_with_already_loaded_permission(): mock_prisma, ): with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: - with patch( - "litellm.proxy.auth.auth_checks.get_object_permission" - ) as mock_get_perm: + with patch("litellm.proxy.auth.auth_checks.get_object_permission") as mock_get_perm: mock_get_team.return_value = mock_team_obj # Call the method - result = await MCPRequestHandler._get_team_object_permission( - mock_user_auth - ) + result = await MCPRequestHandler._get_team_object_permission(mock_user_auth) # Assert we got the object permission assert result == mock_object_permission @@ -3272,9 +3197,7 @@ async def test_get_team_object_permission_ui_session_team_skips_db_lookup(): mock_prisma = MagicMock() with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: - result = await MCPRequestHandler._get_team_object_permission( - mock_user_auth - ) + result = await MCPRequestHandler._get_team_object_permission(mock_user_auth) assert result is None mock_get_team.assert_not_called() @@ -3342,9 +3265,7 @@ async def test_get_allowed_tools_for_server_ui_session_team_keeps_key_restrictio detail={"error": "Team doesn't exist in db. Team=litellm-dashboard."}, ), ): - with patch.object( - MCPRequestHandler, "_get_key_object_permission", return_value=key_perm - ): + with patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_perm): result = await MCPRequestHandler.get_allowed_tools_for_server( server_id="server_1", user_api_key_auth=user_api_key_auth, @@ -3410,9 +3331,7 @@ async def test_get_allowed_mcp_servers_for_team_uses_helper(): return_value=["group-server1", "group-server2"], ) as mock_get_access_group_servers, ): - result = await MCPRequestHandler._get_allowed_mcp_servers_for_team( - mock_user_auth - ) + result = await MCPRequestHandler._get_allowed_mcp_servers_for_team(mock_user_auth) assert set(result) == { "direct-server1", @@ -3455,9 +3374,7 @@ async def test_get_allowed_mcp_servers_for_team_with_no_object_permission(): return_value=mock_team, ), ): - result = await MCPRequestHandler._get_allowed_mcp_servers_for_team( - mock_user_auth - ) + result = await MCPRequestHandler._get_allowed_mcp_servers_for_team(mock_user_auth) assert result == [] @@ -3507,9 +3424,7 @@ async def test_get_allowed_mcp_servers_for_team_without_team_id_returns_empty(): ), ], ) -async def test_get_allowed_mcp_servers_for_key_guard_conditions( - user_api_key_auth, prisma_client_value, scenario -): +async def test_get_allowed_mcp_servers_for_key_guard_conditions(user_api_key_auth, prisma_client_value, scenario): """Ensure guard clauses return [] before hitting get_object_permission.""" with patch( @@ -3517,9 +3432,7 @@ async def test_get_allowed_mcp_servers_for_key_guard_conditions( new_callable=AsyncMock, ) as mock_get_perm: with patch("litellm.proxy.proxy_server.prisma_client", prisma_client_value): - result = await MCPRequestHandler._get_allowed_mcp_servers_for_key( - user_api_key_auth - ) + result = await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_auth) assert result == [] mock_get_perm.assert_not_called() @@ -3546,9 +3459,7 @@ async def test_get_allowed_mcp_servers_for_key_returns_empty_when_db_returns_non ): mock_get_perm.return_value = None - result = await MCPRequestHandler._get_allowed_mcp_servers_for_key( - user_api_key_auth - ) + result = await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_auth) assert result == [] mock_get_perm.assert_awaited_once() @@ -3591,14 +3502,10 @@ async def test_get_allowed_mcp_servers_for_key_prefers_in_memory_permission(): "litellm.proxy.auth.auth_checks.get_object_permission", new_callable=AsyncMock, ) as mock_get_perm: - with patch.object( - MCPRequestHandler, "_get_mcp_servers_from_access_groups" - ) as mock_access_groups: + with patch.object(MCPRequestHandler, "_get_mcp_servers_from_access_groups") as mock_access_groups: mock_access_groups.return_value = ["group-server"] - result = await MCPRequestHandler._get_allowed_mcp_servers_for_key( - user_api_key_auth - ) + result = await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_auth) assert set(result) == {"direct-server", "group-server"} mock_get_perm.assert_not_called() @@ -3619,21 +3526,13 @@ class TestAgentMCPPermissions: team_id="test-team", agent_id="agent-123", ) - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_key" - ) as mock_key: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_team" - ) as mock_team: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_agent" - ) as mock_agent: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key") as mock_key: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team") as mock_team: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_agent") as mock_agent: mock_key.return_value = ["server_1", "server_2"] mock_team.return_value = [] mock_agent.return_value = ["server_1"] - result = await MCPRequestHandler.get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth - ) + result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=user_api_key_auth) assert sorted(result) == ["server_1"] mock_agent.assert_called_once_with(user_api_key_auth) @@ -3644,21 +3543,13 @@ class TestAgentMCPPermissions: user_id="test-user", agent_id="agent-456", ) - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_key" - ) as mock_key: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_team" - ) as mock_team: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_agent" - ) as mock_agent: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key") as mock_key: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team") as mock_team: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_agent") as mock_agent: mock_key.return_value = ["server_1", "server_2"] mock_team.return_value = [] mock_agent.return_value = [] # no agent-level restriction - result = await MCPRequestHandler.get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth - ) + result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=user_api_key_auth) assert sorted(result) == ["server_1", "server_2"] mock_agent.assert_called_once_with(user_api_key_auth) @@ -3669,21 +3560,13 @@ class TestAgentMCPPermissions: user_id="test-user", agent_id="agent-789", ) - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_key" - ) as mock_key: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_team" - ) as mock_team: - with patch.object( - MCPRequestHandler, "_get_allowed_mcp_servers_for_agent" - ) as mock_agent: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_key") as mock_key: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_team") as mock_team: + with patch.object(MCPRequestHandler, "_get_allowed_mcp_servers_for_agent") as mock_agent: mock_key.return_value = ["server_1", "server_2"] mock_team.return_value = [] mock_agent.return_value = ["server_2", "server_3"] - result = await MCPRequestHandler.get_allowed_mcp_servers( - user_api_key_auth=user_api_key_auth - ) + result = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth=user_api_key_auth) assert sorted(result) == ["server_2"] async def test_get_allowed_tools_for_server_agent_intersection(self): @@ -3696,9 +3579,7 @@ class TestAgentMCPPermissions: key_perm = MagicMock() key_perm.mcp_tool_permissions = {"server_1": ["tool_a", "tool_b"]} team_perm = None - with patch.object( - MCPRequestHandler, "_get_key_object_permission", return_value=key_perm - ): + with patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_perm): with patch.object( MCPRequestHandler, "_get_team_object_permission", @@ -3730,9 +3611,7 @@ class TestAgentMCPPermissions: ) key_perm = MagicMock() key_perm.mcp_tool_permissions = {"server_1": ["tool_a", "tool_b"]} - with patch.object( - MCPRequestHandler, "_get_key_object_permission", return_value=key_perm - ): + with patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_perm): with patch.object( MCPRequestHandler, "_get_team_object_permission", @@ -3762,9 +3641,7 @@ class TestAgentMCPPermissions: agent_row = MagicMock() agent_row.object_permission_id = "perm-xyz" prisma_client = MagicMock() - prisma_client.db.litellm_agentstable.find_unique = AsyncMock( - return_value=agent_row - ) + prisma_client.db.litellm_agentstable.find_unique = AsyncMock(return_value=agent_row) user_api_key_auth = UserAPIKeyAuth( api_key="test-key", user_id="test-user", @@ -3782,9 +3659,7 @@ class TestAgentMCPPermissions: return_value=expected_perm, ) as mock_get_perm, ): - result = await MCPRequestHandler._get_agent_object_permission( - user_api_key_auth - ) + result = await MCPRequestHandler._get_agent_object_permission(user_api_key_auth) assert result is expected_perm mock_get_perm.assert_awaited_once() assert mock_get_perm.await_args.kwargs["object_permission_id"] == "perm-xyz" @@ -3804,9 +3679,7 @@ class TestAgentMCPPermissions: agent_row = MagicMock() agent_row.object_permission_id = None prisma_client = MagicMock() - prisma_client.db.litellm_agentstable.find_unique = AsyncMock( - return_value=agent_row - ) + prisma_client.db.litellm_agentstable.find_unique = AsyncMock(return_value=agent_row) user_api_key_auth = UserAPIKeyAuth( api_key="test-key", user_id="test-user", @@ -3822,14 +3695,8 @@ class TestAgentMCPPermissions: new_callable=AsyncMock, ) as mock_get_perm, ): - assert ( - await MCPRequestHandler._get_agent_object_permission(user_api_key_auth) - is None - ) - assert ( - await MCPRequestHandler._get_agent_object_permission(user_api_key_auth) - is None - ) + assert await MCPRequestHandler._get_agent_object_permission(user_api_key_auth) is None + assert await MCPRequestHandler._get_agent_object_permission(user_api_key_auth) is None mock_get_perm.assert_not_awaited() prisma_client.db.litellm_agentstable.find_unique.assert_awaited_once() @@ -3870,9 +3737,7 @@ async def test_tool_permission_servers_included_in_allowed_servers(): ) with ( - patch.object( - MCPRequestHandler, "_get_key_object_permission", return_value=perm - ), + patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=perm), patch.object( MCPRequestHandler, "_get_mcp_servers_from_access_groups", @@ -4105,9 +3970,7 @@ class TestOrgMCPPermissions: org_perm.mcp_tool_permissions = {"server_1": ["tool_a", "tool_b"]} with ( - patch.object( - MCPRequestHandler, "_get_key_object_permission", return_value=key_perm - ), + patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_perm), patch.object( MCPRequestHandler, "_get_team_object_permission", @@ -4137,9 +4000,7 @@ class TestOrgMCPPermissions: org_perm.mcp_tool_permissions = {} with ( - patch.object( - MCPRequestHandler, "_get_key_object_permission", return_value=key_perm - ), + patch.object(MCPRequestHandler, "_get_key_object_permission", return_value=key_perm), patch.object( MCPRequestHandler, "_get_team_object_permission", @@ -4231,9 +4092,7 @@ async def test_mcp_key_access_group_extras_when_team_authorized(): ] _start_patches(patches) try: - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - valid_token - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(valid_token) assert result == ["srv-stripe"] finally: _stop_patches(patches) @@ -4270,9 +4129,7 @@ async def test_mcp_key_access_group_extras_when_key_directly_authorized(): ] _start_patches(patches) try: - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - valid_token - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(valid_token) assert result == ["srv-stripe"] finally: _stop_patches(patches) @@ -4286,9 +4143,7 @@ async def test_mcp_key_access_group_extras_when_key_has_no_groups(): access_group_ids=[], team_id="team-a", ) - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - valid_token - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(valid_token) assert result == [] @@ -4315,9 +4170,7 @@ async def test_mcp_key_access_group_extras_when_group_has_no_servers(): ] _start_patches(patches) try: - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - valid_token - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(valid_token) assert result == [] finally: _stop_patches(patches) @@ -4351,9 +4204,7 @@ async def test_mcp_key_access_group_extras_granted_even_when_group_authorizes_ne ] _start_patches(patches) try: - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - valid_token - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(valid_token) assert result == ["srv-finance-only"] finally: _stop_patches(patches) @@ -4376,9 +4227,7 @@ async def test_mcp_key_access_group_extras_when_get_access_object_raises(): ] _start_patches(patches) try: - result = await MCPRequestHandler._get_key_access_group_mcp_server_extras( - valid_token - ) + result = await MCPRequestHandler._get_key_access_group_mcp_server_extras(valid_token) assert result == [] finally: _stop_patches(patches) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_adapter.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_adapter.py index fbe768e07c6..17960e917a4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_adapter.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_adapter.py @@ -23,6 +23,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import ( AuthorizationCodeConfig, CredError, NoneConfig, + PassthroughConfig, SharedKey, TokenExchangeConfig, ) @@ -234,6 +235,12 @@ def test_token_exchange_empty_subject_token_type_normalizes_to_default(): assert spec.config.subject_token_type == "urn:ietf:params:oauth:token-type:access_token" +@pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) +def test_client_forwarded_modes_map_to_passthrough_config(auth_type): + spec = to_server_spec(_server(auth_type=auth_type)) + assert spec is not None and isinstance(spec.config, PassthroughConfig) + + @pytest.mark.parametrize( "server", [ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py index f8e45b38b49..64226eea821 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_resolver.py @@ -1,9 +1,9 @@ """Tests for the resolver dispatch: live arms produce auth, stubbed arms fail closed. -`none`, `api_key` (shared-key source), `authorization_code`, and `token_exchange` are implemented; -every other arm, plus the `api_key` BYOK source, returns a typed `not_implemented` error until its -mode lands. Parametrizing the stubs over one config each also guards reachability: a dropped `case` -would hit `assert_never` and raise instead of returning the stub. +`none`, `api_key` (shared-key source), `passthrough`, `authorization_code`, and `token_exchange` are +implemented; every other arm, plus the `api_key` BYOK source, returns a typed `not_implemented` error +until its mode lands. Parametrizing the stubs over one config each also guards reachability: a dropped +`case` would hit `assert_never` and raise instead of returning the stub. """ import httpx @@ -253,9 +253,24 @@ async def test_token_exchange_without_an_exchanger_fails_closed(): assert result.error.tag == "misconfigured" +@pytest.mark.asyncio +async def test_passthrough_forwards_the_inbound_token_verbatim(): + subject = Subject(tenant_id="", subject_id="", inbound_token=SecretStr("Bearer upstream-xyz")) + result = await UpstreamCredentialProvider().resolve_credentials(subject, _spec(PassthroughConfig())) + assert isinstance(result, Ok) + assert isinstance(result.ok, StaticHeaderAuth) + assert _emitted(result.ok)["Authorization"] == "Bearer upstream-xyz" + + +@pytest.mark.asyncio +async def test_passthrough_without_inbound_token_is_a_no_op(): + result = await UpstreamCredentialProvider().resolve_credentials(_SUBJECT, _spec(PassthroughConfig())) + assert isinstance(result, Ok) + assert isinstance(result.ok, NoOpAuth) + + _STUBBED = [ ("api_key_byok", ApiKeyConfig(key_source=Byok())), - ("passthrough", PassthroughConfig()), ("client_credentials", ClientCredentialsConfig()), ("aws_sigv4", AwsSigV4Config(region="us-east-1")), ] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 6fa69b2f96c..a7a0379c7ad 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -1267,6 +1267,229 @@ class TestMCPServerManager: assert captured_extra_headers == {"Authorization": "Bearer upstream-oauth-bearer"} + async def _capture_call_extra_headers(self, server, oauth2_headers, raw_headers, user_api_key_auth): + manager = MCPServerManager() + mock_client = AsyncMock() + mock_client.call_tool = AsyncMock(return_value=CallToolResult(content=[], isError=False)) + captured = {"extra_headers": "unset"} + + async def capture_create_mcp_client( + server, mcp_auth_header, extra_headers, stdio_env, subject_token=None, **kwargs + ): # pragma: no cover - helper + captured["extra_headers"] = extra_headers + return mock_client + + manager._create_mcp_client = AsyncMock(side_effect=capture_create_mcp_client) + await manager._call_regular_mcp_tool( + mcp_server=server, + original_tool_name="tool", + arguments={}, + tasks=[], + mcp_auth_header=None, + mcp_server_auth_headers=None, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + proxy_logging_obj=None, + user_api_key_auth=user_api_key_auth, + ) + return captured["extra_headers"] + + @pytest.mark.asyncio + async def test_call_regular_mcp_tool_true_passthrough_forwards_authorization(self): + from litellm.proxy._types import UserAPIKeyAuth + + server = MCPServer( + server_id="server-true-passthrough", + name="tp-server", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.true_passthrough, + ) + extra_headers = await self._capture_call_extra_headers( + server, + oauth2_headers={"Authorization": "Bearer upstream-token"}, + raw_headers={"authorization": "Bearer upstream-token"}, + user_api_key_auth=UserAPIKeyAuth(api_key=None), + ) + assert extra_headers == {"Authorization": "Bearer upstream-token"} + + @pytest.mark.asyncio + async def test_call_regular_mcp_tool_oauth_delegate_forwards_separate_authorization(self): + from litellm.proxy._types import UserAPIKeyAuth + + server = MCPServer( + server_id="server-oauth-delegate", + name="od-server", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + extra_headers = await self._capture_call_extra_headers( + server, + oauth2_headers={"Authorization": "Bearer upstream-token"}, + raw_headers={ + "x-litellm-api-key": "Bearer sk-litellm-key", + "authorization": "Bearer upstream-token", + }, + user_api_key_auth=UserAPIKeyAuth(api_key="sk-litellm-key"), + ) + assert extra_headers == {"Authorization": "Bearer upstream-token"} + + @pytest.mark.asyncio + async def test_call_regular_mcp_tool_oauth_delegate_never_forwards_admission_key(self): + from litellm.proxy._types import UserAPIKeyAuth + + server = MCPServer( + server_id="server-oauth-delegate-leak", + name="od-server", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + extra_headers = await self._capture_call_extra_headers( + server, + oauth2_headers={"Authorization": "Bearer sk-litellm-key"}, + raw_headers={"authorization": "Bearer sk-litellm-key"}, + user_api_key_auth=UserAPIKeyAuth(api_key="sk-litellm-key"), + ) + assert not extra_headers or "authorization" not in {k.lower() for k in extra_headers} + + def test_should_strip_caller_authorization_new_modes(self): + from litellm.proxy._types import UserAPIKeyAuth + + true_passthrough = MCPServer( + server_id="tp", + name="tp", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.true_passthrough, + ) + assert ( + _should_strip_caller_authorization( + mcp_server=true_passthrough, + raw_headers={"authorization": "Bearer upstream"}, + user_api_key_auth=UserAPIKeyAuth(api_key=None), + ) + is False + ) + + oauth_delegate = MCPServer( + server_id="od", + name="od", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + assert ( + _should_strip_caller_authorization( + mcp_server=oauth_delegate, + raw_headers={ + "x-litellm-api-key": "Bearer sk-litellm-key", + "authorization": "Bearer upstream", + }, + user_api_key_auth=UserAPIKeyAuth(api_key="sk-litellm-key"), + ) + is False + ) + assert ( + _should_strip_caller_authorization( + mcp_server=oauth_delegate, + raw_headers={"authorization": "Bearer sk-litellm-key"}, + user_api_key_auth=UserAPIKeyAuth(api_key="sk-litellm-key"), + ) + is True + ) + + def test_should_strip_authorization_for_oauth_delegate_admitted_via_jwt_without_api_key(self): + """JWT / SSO / OIDC / session admission yields a UserAPIKeyAuth with a user_id but + api_key=None; the caller's Authorization was that credential and must be stripped for + oauth_delegate when no separate x-litellm-api-key carried admission (LIT-3794-class leak).""" + from litellm.proxy._types import UserAPIKeyAuth + + oauth_delegate = MCPServer( + server_id="od-jwt", + name="od", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + assert ( + _should_strip_caller_authorization( + mcp_server=oauth_delegate, + raw_headers={"authorization": "Bearer eyJ-idp-jwt"}, + user_api_key_auth=UserAPIKeyAuth(user_id="alice", api_key=None), + ) + is True + ) + assert ( + _should_strip_caller_authorization( + mcp_server=oauth_delegate, + raw_headers={ + "x-litellm-api-key": "Bearer sk-1234", + "authorization": "Bearer upstream", + }, + user_api_key_auth=UserAPIKeyAuth(user_id="alice", api_key=None), + ) + is False + ) + + @pytest.mark.asyncio + async def test_call_regular_mcp_tool_oauth_delegate_never_forwards_jwt_admission(self): + from litellm.proxy._types import UserAPIKeyAuth + + server = MCPServer( + server_id="od-jwt-e2e", + name="od", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + extra_headers = await self._capture_call_extra_headers( + server, + oauth2_headers={"Authorization": "Bearer eyJ-idp-jwt"}, + raw_headers={"authorization": "Bearer eyJ-idp-jwt"}, + user_api_key_auth=UserAPIKeyAuth(user_id="alice", api_key=None), + ) + assert not extra_headers or "authorization" not in {k.lower() for k in extra_headers} + + def test_new_passthrough_modes_require_per_user_auth(self): + for auth_type in (MCPAuth.true_passthrough, MCPAuth.oauth_delegate): + server = MCPServer( + server_id="s", + name="s", + url="https://example.com", + transport=MCPTransport.http, + auth_type=auth_type, + ) + assert server.requires_per_user_auth is True + + @pytest.mark.asyncio + async def test_create_mcp_client_forwarded_modes_use_the_passthrough_arm(self): + manager = MCPServerManager() + server = MCPServer( + server_id="tp-egress", + name="tp", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.true_passthrough, + ) + with ( + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.resolve_mcp_auth", + new_callable=AsyncMock, + ) as mock_resolve, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as mock_client_cls, + ): + await manager._create_mcp_client(server=server, extra_headers={"Authorization": "Bearer upstream-token"}) + mock_resolve.assert_not_awaited() + kwargs = mock_client_cls.call_args.kwargs + emitted = httpx.Request("GET", "https://example.com/mcp") + flow = kwargs["resolved_auth"].auth_flow(emitted) + next(flow) + flow.close() + assert emitted.headers["Authorization"] == "Bearer upstream-token" + assert not kwargs["extra_headers"] or "authorization" not in {k.lower() for k in kwargs["extra_headers"]} + @pytest.mark.asyncio async def test_get_prompts_from_server_success(self): """Ensure prompts are fetched and prefixed when requested.""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d9bba85bf4b..9765775af8f 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27008,7 +27008,7 @@ export interface components { /** Alias */ alias?: string | null; /** Auth Type */ - auth_type?: ("none" | "api_key" | "bearer_token" | "basic" | "authorization" | "oauth2" | "aws_sigv4" | "token" | "oauth2_token_exchange") | null; + auth_type?: ("none" | "api_key" | "bearer_token" | "basic" | "authorization" | "oauth2" | "aws_sigv4" | "token" | "oauth2_token_exchange" | "true_passthrough" | "oauth_delegate") | null; /** Mcp Info */ mcp_info?: { [key: string]: unknown; From 446a6e8cddc883f058023ec3a173115d87437b27 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 6 Jul 2026 13:23:28 -0700 Subject: [PATCH 002/101] fix(mcp): route true_passthrough and oauth_delegate through upstream OAuth discovery Both modes advertised LiteLLM as the authorization server and answered initialize locally, so a client with no token connected empty and was never driven into the upstream OAuth flow. The protected-resource discovery now proxies the upstream metadata for both modes (verbatim for true_passthrough, resource rewritten to the gateway for oauth_delegate), and the preemptive 401 emits the matching challenge: oauth_delegate uses the gateway-proxied resource_metadata once admission passes, true_passthrough probes the upstream anonymously and surfaces its WWW-Authenticate verbatim so the client authorizes directly against the upstream --- .../mcp_server/discoverable_endpoints.py | 9 +- .../proxy/_experimental/mcp_server/server.py | 27 +- .../mcp_server/test_mcp_oauth_passthrough.py | 99 ++++++ .../mcp_server/test_mcp_stale_session.py | 310 ++++++++++++++++++ 4 files changed, 440 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 89d645b6f8a..f383c99c563 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1337,7 +1337,9 @@ async def _build_oauth_protected_resource_response( # Pass-through branch: proxy the upstream's own metadata so discovery # directs the client at the real IdP (Okta, Keycloak, …) instead of us. - if mcp_server is not None and mcp_server.is_oauth_passthrough: + if mcp_server is not None and ( + mcp_server.is_oauth_passthrough or mcp_server.is_oauth_delegate or mcp_server.is_true_passthrough + ): try: upstream_metadata = await fetch_upstream_oauth_protected_resource(mcp_server) except Exception as exc: @@ -1353,8 +1355,9 @@ async def _build_oauth_protected_resource_response( ) if upstream_metadata is not None: - response = {**upstream_metadata, "resource": resource_url} - return response + if mcp_server.is_true_passthrough: + return upstream_metadata + return {**upstream_metadata, "resource": resource_url} # Upstream responded but with non-200 or non-dict payload. For # pass-through servers the gateway is NOT the authorization server, diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 96fd014fbe1..7a854f698c9 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -3526,6 +3526,29 @@ if MCP_AVAILABLE: headers={"www-authenticate": www_authenticate}, ) + if server and server.is_oauth_delegate and _get_forwarded_auth_from_scope(scope) is None: + www_authenticate = _get_passthrough_www_authenticate( + scope=scope, + server_name=server_name, + ) + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"www-authenticate": www_authenticate}, + ) + + if server and server.is_true_passthrough and not _scope_has_authorization_header(scope): + upstream_status, upstream_www_authenticate = await _probe_upstream_auth(server.url or "", "") + if upstream_status == 401 and upstream_www_authenticate: + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"www-authenticate": upstream_www_authenticate}, + ) + + def _scope_has_authorization_header(scope: Scope) -> bool: + return any(key.lower() == b"authorization" for key, _ in scope.get("headers", [])) + def _get_forwarded_auth_from_scope(scope: Scope) -> Optional[str]: """Return the upstream-bound ``Authorization`` header value, or None. @@ -3553,7 +3576,7 @@ if MCP_AVAILABLE: url: str, auth_header: str, timeout: float = 5.0, - ) -> tuple: + ) -> tuple[int, Optional[str]]: """JSON-RPC initialize-probe the upstream URL to check whether the token is accepted. Uses POST so StreamableHTTP MCP servers run the same auth path as a @@ -3583,8 +3606,8 @@ if MCP_AVAILABLE: }, } probe_headers = { - "Authorization": auth_header, "Accept": "application/json, text/event-stream", + **({"Authorization": auth_header} if auth_header else {}), } try: resp = await client.post( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py index ad78609ee18..002e98034c1 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py @@ -472,3 +472,102 @@ async def test_oauth_protected_resource_gateway_managed_unchanged(): "https://gateway.example.com/keycloak_whoami" ] assert result["scopes_supported"] == ["read"] + + +def _make_upstream_metadata_client() -> tuple[dict, MagicMock]: + upstream_payload = { + "resource": "https://upstream.example.com/mcp", + "authorization_servers": ["https://okta.example.com/oauth2/default"], + "scopes_supported": ["openid", "profile"], + "bearer_methods_supported": ["header"], + } + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = upstream_payload + mock_client = MagicMock() + mock_client.get = AsyncMock(return_value=mock_response) + return upstream_payload, mock_client + + +@pytest.mark.asyncio +async def test_oauth_protected_resource_oauth_delegate_proxies_upstream_with_gateway_resource(): + """oauth_delegate discovery must proxy the upstream's authorization_servers + (so the client authorizes against the upstream IdP) while rewriting resource + to the gateway (so bearers are presented back to LiteLLM). A regression that + dropped oauth_delegate from the pass-through predicate would fall through to + the gateway-AS branch and advertise LiteLLM as the authorization server.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + global_mcp_server_manager.registry.clear() + delegate_server = MCPServer( + server_id="delegate-1", + name="sample_docs", + server_name="sample_docs", + alias="sample_docs", + url="https://upstream.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + global_mcp_server_manager.registry[delegate_server.server_id] = delegate_server + + _, mock_client = _make_upstream_metadata_client() + try: + with patch.object( + discoverable_endpoints, "get_async_httpx_client", return_value=mock_client + ): + result = await _build_oauth_protected_resource_response( + request=_make_request(), + mcp_server_name="sample_docs", + use_standard_pattern=True, + ) + + assert result["authorization_servers"] == [ + "https://okta.example.com/oauth2/default" + ] + assert result["resource"].endswith("/mcp/sample_docs") + assert result["resource"] != "https://upstream.example.com/mcp" + finally: + global_mcp_server_manager.registry.clear() + + +@pytest.mark.asyncio +async def test_oauth_protected_resource_true_passthrough_returns_upstream_metadata_verbatim(): + """true_passthrough discovery must return the upstream metadata verbatim, + resource included, so the client treats the upstream as the resource and + authorizes directly against it. A regression that rewrote resource (the + gateway-proxied behavior) would break the transparent-proxy contract.""" + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + global_mcp_server_manager.registry.clear() + true_passthrough_server = MCPServer( + server_id="tp-1", + name="sample_docs", + server_name="sample_docs", + alias="sample_docs", + url="https://upstream.example.com/mcp", + transport=MCPTransport.http, + auth_type=MCPAuth.true_passthrough, + ) + global_mcp_server_manager.registry[true_passthrough_server.server_id] = ( + true_passthrough_server + ) + + upstream_payload, mock_client = _make_upstream_metadata_client() + try: + with patch.object( + discoverable_endpoints, "get_async_httpx_client", return_value=mock_client + ): + result = await _build_oauth_protected_resource_response( + request=_make_request(), + mcp_server_name="sample_docs", + use_standard_pattern=True, + ) + + assert result == upstream_payload + assert result["resource"] == "https://upstream.example.com/mcp" + finally: + global_mcp_server_manager.registry.clear() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index 76614a35b53..44ea5f43a70 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -713,6 +713,9 @@ async def test_handle_streamable_http_mcp_delegated_server_surfaces_upstream_cha delegated_server.auth_type = MCPAuth.oauth2 delegated_server.delegate_auth_to_upstream = True delegated_server.needs_user_oauth_token = True + delegated_server.is_oauth_passthrough = False + delegated_server.is_oauth_delegate = False + delegated_server.is_true_passthrough = False delegated_server.server_id = "delegated-oauth-server" upstream_challenge = 'Bearer resource_metadata="https://upstream.example.com/.well-known/oauth-protected-resource"' @@ -1048,3 +1051,310 @@ async def test_handle_streamable_http_mcp_token_exchange_without_subject_returns assert "/.well-known/oauth-protected-resource" in challenge assert challenge.split('resource_metadata="', 1)[1].split('"', 1)[0].endswith("/mcp/obo_server") assert 'error="invalid_token"' in challenge + + +def _passthrough_mode_scope(server_name: str, extra_headers=None): + headers = [ + (b"content-type", b"application/json"), + (b"host", b"litellm.example.com"), + ] + list(extra_headers or []) + return { + "type": "http", + "method": "POST", + "path": f"/mcp/{server_name}", + "_original_path": f"/{server_name}/mcp", + "scheme": "https", + "query_string": b"", + "root_path": "", + "server": ("litellm.example.com", 443), + "headers": headers, + } + + +def _build_passthrough_mode_server(server_name: str, auth_type): + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + return MCPServer( + server_id=f"{server_name}-id", + name=server_name, + server_name=server_name, + alias=server_name, + url="https://upstream.example.com/mcp", + transport=MCPTransport.http, + auth_type=auth_type, + ) + + +@pytest.mark.asyncio +async def test_handle_streamable_http_mcp_oauth_delegate_without_token_returns_gateway_proxied_401(): + """oauth_delegate is admitted with the LiteLLM key but still owns upstream + OAuth. With no forwarded upstream token the gateway must challenge with the + proxied resource_metadata (which advertises the upstream IdP), never the + gateway authorization_uri and never a silent 200.""" + from fastapi import HTTPException + + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateful, + ) + except ImportError: + pytest.skip("MCP server not available") + + scope = _passthrough_mode_scope("od_server") + receive = AsyncMock( + return_value={ + "type": "http.request", + "body": b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}', + "more_body": False, + } + ) + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = "u1" + od_server = _build_passthrough_mode_server("od_server", MCPAuth.oauth_delegate) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, ["od_server"], None, None, None), + ), + patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=od_server, + ), + patch.object( + session_manager_stateful, + "handle_request", + new_callable=AsyncMock, + ) as mock_handle_request, + ): + with pytest.raises(HTTPException) as exc_info: + await handle_streamable_http_mcp(scope, receive, send) + + assert mock_handle_request.await_count == 0 + assert exc_info.value.status_code == 401 + challenge = exc_info.value.headers["www-authenticate"] + assert "resource_metadata=" in challenge + assert "authorization_uri=" not in challenge + assert "/.well-known/oauth-protected-resource/od_server/mcp" in challenge + + +@pytest.mark.asyncio +async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_skips_challenge(): + """When the oauth_delegate caller carries both the LiteLLM key and a separate + upstream Authorization, the gateway must forward to the session manager, not + re-challenge. Guards the ``_get_forwarded_auth_from_scope(...) is None`` + condition: dropping it would 401 even a fully-authenticated request.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateless, + ) + except ImportError: + pytest.skip("MCP server not available") + + scope = _passthrough_mode_scope( + "od_server", + extra_headers=[ + (b"x-litellm-api-key", b"Bearer sk-1234"), + (b"authorization", b"Bearer upstream-token"), + ], + ) + receive = AsyncMock( + return_value={ + "type": "http.request", + "body": b'{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}', + "more_body": False, + } + ) + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = "u1" + od_server = _build_passthrough_mode_server("od_server", MCPAuth.oauth_delegate) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, ["od_server"], None, None, None), + ), + patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + new_callable=AsyncMock, + return_value=False, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=od_server, + ), + patch.object( + session_manager_stateless, + "handle_request", + new_callable=AsyncMock, + ) as mock_handle_request, + patch.object(session_manager_stateless, "_server_instances", {}), + ): + await handle_streamable_http_mcp(scope, receive, send) + + assert mock_handle_request.await_count == 1 + + +@pytest.mark.asyncio +async def test_handle_streamable_http_mcp_true_passthrough_without_token_surfaces_verbatim_upstream_challenge(): + """true_passthrough is a transparent proxy: with no client Authorization the + gateway probes the upstream and surfaces its own WWW-Authenticate verbatim, + so the client discovers and authorizes against the upstream directly. Guards + against answering initialize locally with a silent 200.""" + from fastapi import HTTPException + + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateful, + ) + except ImportError: + pytest.skip("MCP server not available") + + upstream_challenge = 'Bearer resource_metadata="https://upstream.example.com/.well-known/oauth-protected-resource"' + probe_response = MagicMock() + probe_response.status_code = 401 + probe_response.headers = {"www-authenticate": upstream_challenge} + probe_client = MagicMock() + probe_client.post = AsyncMock(return_value=probe_response) + + scope = _passthrough_mode_scope("tp_server") + receive = AsyncMock( + return_value={ + "type": "http.request", + "body": b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}', + "more_body": False, + } + ) + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = None + tp_server = _build_passthrough_mode_server("tp_server", MCPAuth.true_passthrough) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, ["tp_server"], None, None, None), + ), + patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + return_value=probe_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=tp_server, + ), + patch.object( + session_manager_stateful, + "handle_request", + new_callable=AsyncMock, + ) as mock_handle_request, + ): + with pytest.raises(HTTPException) as exc_info: + await handle_streamable_http_mcp(scope, receive, send) + + assert mock_handle_request.await_count == 0 + assert exc_info.value.status_code == 401 + assert exc_info.value.headers["www-authenticate"] == upstream_challenge + probe_client.post.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_handle_streamable_http_mcp_true_passthrough_with_token_skips_probe_and_challenge(): + """When the true_passthrough caller already carries an Authorization the + gateway must forward without probing or challenging. Guards the + ``not _scope_has_authorization_header(scope)`` condition and the no-probe + fast path.""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateless, + ) + except ImportError: + pytest.skip("MCP server not available") + + probe_client = MagicMock() + probe_client.post = AsyncMock() + + scope = _passthrough_mode_scope( + "tp_server", + extra_headers=[(b"authorization", b"Bearer upstream-token")], + ) + receive = AsyncMock( + return_value={ + "type": "http.request", + "body": b'{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}', + "more_body": False, + } + ) + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = None + tp_server = _build_passthrough_mode_server("tp_server", MCPAuth.true_passthrough) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, ["tp_server"], None, None, None), + ), + patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch( + "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", + True, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + new_callable=AsyncMock, + return_value=False, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", + return_value=probe_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=tp_server, + ), + patch.object( + session_manager_stateless, + "handle_request", + new_callable=AsyncMock, + ) as mock_handle_request, + patch.object(session_manager_stateless, "_server_instances", {}), + ): + await handle_streamable_http_mcp(scope, receive, send) + + assert mock_handle_request.await_count == 1 + probe_client.post.assert_not_awaited() From 9bf3907de546beb34bbfddad048c8295e746e50a Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 7 Jul 2026 16:13:43 -0700 Subject: [PATCH 003/101] fix(mcp): bind oauth_delegate discovery resource to the upstream oauth_delegate forwards the caller's token to the upstream, which validates its audience, so the protected-resource metadata must keep resource pointing at the upstream (returned verbatim, like true_passthrough) rather than rewriting it to the gateway. Rewriting to the gateway asks the client to mint a token bound to the gateway audience, which a strict IdP (Entra) refuses to issue for an unregistered resource and a spec-compliant upstream rejects on receipt. The legacy is_oauth_passthrough opt-in keeps the gateway rewrite unchanged. --- .../mcp_server/discoverable_endpoints.py | 16 ++-- .../mcp_server/test_mcp_oauth_passthrough.py | 94 ++++++------------- 2 files changed, 39 insertions(+), 71 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index f383c99c563..415336b1a9e 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1292,11 +1292,15 @@ async def _build_oauth_protected_resource_response( """ Build OAuth protected resource response with the appropriate URL pattern. - For pass-through MCP servers (``MCPServer.is_oauth_passthrough``), the - gateway proxies the upstream's own ``oauth-protected-resource`` metadata - so that standards-compliant MCP clients discover the **upstream** IdP - instead of the gateway. The ``resource`` field is rewritten to the - gateway's own URL so clients present the bearer token back to the gateway. + For pass-through MCP servers, the gateway proxies the upstream's own + ``oauth-protected-resource`` metadata so standards-compliant MCP clients + discover the **upstream** IdP instead of the gateway. For ``true_passthrough`` + and ``oauth_delegate`` the metadata is returned verbatim (``resource`` stays + the upstream): the caller's token is forwarded to and validated by the + upstream, so its audience must be the upstream — rewriting it to the gateway + would make a strict IdP (e.g. Entra) refuse to mint it or the upstream reject + it. Only the legacy ``is_oauth_passthrough`` opt-in rewrites ``resource`` to + the gateway's own URL so clients present the bearer token back to the gateway. Args: request: FastAPI Request object @@ -1355,7 +1359,7 @@ async def _build_oauth_protected_resource_response( ) if upstream_metadata is not None: - if mcp_server.is_true_passthrough: + if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate: return upstream_metadata return {**upstream_metadata, "resource": resource_url} diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py index 002e98034c1..ef2a8318d7a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_oauth_passthrough.py @@ -34,8 +34,7 @@ from litellm.types.mcp_server.mcp_server_manager import MCPServer def _mock_mcp_client_ip(): """Bypass IP-based access control in tests.""" with patch( - "litellm.proxy._experimental.mcp_server.discoverable_endpoints" - ".IPAddressUtils.get_mcp_client_ip", + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.IPAddressUtils.get_mcp_client_ip", return_value=None, ): yield @@ -191,9 +190,7 @@ async def test_oauth_protected_resource_passthrough_proxies_upstream_metadata(): extra_headers=["Authorization"], oauth_passthrough=True, ) - global_mcp_server_manager.registry[passthrough_server.server_id] = ( - passthrough_server - ) + global_mcp_server_manager.registry[passthrough_server.server_id] = passthrough_server upstream_payload = { "resource": "https://upstream.example.com/mcp", @@ -207,18 +204,14 @@ async def test_oauth_protected_resource_passthrough_proxies_upstream_metadata(): mock_client = MagicMock() mock_client.get = AsyncMock(return_value=mock_response) - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): result = await _build_oauth_protected_resource_response( request=_make_request(), mcp_server_name="sample_docs", use_standard_pattern=True, ) - assert result["authorization_servers"] == [ - "https://okta.example.com/oauth2/default" - ] + assert result["authorization_servers"] == ["https://okta.example.com/oauth2/default"] # resource is normalized to the gateway URL so bearers are sent back to us assert result["resource"].endswith("/mcp/sample_docs") assert result["scopes_supported"] == ["openid", "profile"] @@ -242,9 +235,7 @@ async def test_oauth_protected_resource_passthrough_cache_hit(): extra_headers=["Authorization"], oauth_passthrough=True, ) - global_mcp_server_manager.registry[passthrough_server.server_id] = ( - passthrough_server - ) + global_mcp_server_manager.registry[passthrough_server.server_id] = passthrough_server mock_response = MagicMock() mock_response.status_code = 200 @@ -254,9 +245,7 @@ async def test_oauth_protected_resource_passthrough_cache_hit(): mock_client = MagicMock() mock_client.get = AsyncMock(return_value=mock_response) - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): await _build_oauth_protected_resource_response( request=_make_request(), mcp_server_name="sample_docs", @@ -348,12 +337,8 @@ async def test_oauth_metadata_cache_expired_entry_is_refetched(): mock_client = MagicMock() mock_client.get = AsyncMock(return_value=mock_response) - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): - result = await discoverable_endpoints.fetch_upstream_oauth_protected_resource( - passthrough_server - ) + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): + result = await discoverable_endpoints.fetch_upstream_oauth_protected_resource(passthrough_server) assert result == {"authorization_servers": ["https://fresh.example.com"]} assert mock_client.get.await_count == 1 @@ -377,16 +362,12 @@ async def test_oauth_protected_resource_passthrough_network_error_returns_502(): extra_headers=["Authorization"], oauth_passthrough=True, ) - global_mcp_server_manager.registry[passthrough_server.server_id] = ( - passthrough_server - ) + global_mcp_server_manager.registry[passthrough_server.server_id] = passthrough_server mock_client = MagicMock() mock_client.get = AsyncMock(side_effect=httpx.ConnectError("boom")) - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): with pytest.raises(HTTPException) as exc_info: await _build_oauth_protected_resource_response( request=_make_request(), @@ -414,16 +395,10 @@ async def test_fetch_upstream_metadata_returns_none_when_not_all_candidates_netw not_found_response = MagicMock() not_found_response.status_code = 404 mock_client = MagicMock() - mock_client.get = AsyncMock( - side_effect=[not_found_response, httpx.ConnectError("path fallback failed")] - ) + mock_client.get = AsyncMock(side_effect=[not_found_response, httpx.ConnectError("path fallback failed")]) - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): - result = await discoverable_endpoints.fetch_upstream_oauth_protected_resource( - passthrough_server - ) + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): + result = await discoverable_endpoints.fetch_upstream_oauth_protected_resource(passthrough_server) assert result is None assert mock_client.get.await_count == 2 @@ -458,9 +433,7 @@ async def test_oauth_protected_resource_gateway_managed_unchanged(): mock_client = MagicMock() mock_client.get = AsyncMock() - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): result = await _build_oauth_protected_resource_response( request=_make_request(), mcp_server_name="keycloak_whoami", @@ -468,9 +441,7 @@ async def test_oauth_protected_resource_gateway_managed_unchanged(): ) mock_client.get.assert_not_awaited() - assert result["authorization_servers"] == [ - "https://gateway.example.com/keycloak_whoami" - ] + assert result["authorization_servers"] == ["https://gateway.example.com/keycloak_whoami"] assert result["scopes_supported"] == ["read"] @@ -490,12 +461,13 @@ def _make_upstream_metadata_client() -> tuple[dict, MagicMock]: @pytest.mark.asyncio -async def test_oauth_protected_resource_oauth_delegate_proxies_upstream_with_gateway_resource(): - """oauth_delegate discovery must proxy the upstream's authorization_servers - (so the client authorizes against the upstream IdP) while rewriting resource - to the gateway (so bearers are presented back to LiteLLM). A regression that - dropped oauth_delegate from the pass-through predicate would fall through to - the gateway-AS branch and advertise LiteLLM as the authorization server.""" +async def test_oauth_protected_resource_oauth_delegate_returns_upstream_metadata_verbatim(): + """oauth_delegate discovery must return the upstream metadata verbatim, + resource included. The caller's token is forwarded to and validated by the + upstream, so its audience must be the upstream; rewriting resource to the + gateway would make a strict IdP refuse to mint it or the upstream reject it. + A regression that dropped oauth_delegate from the pass-through predicate would + fall through to the gateway-AS branch and advertise LiteLLM as the AS.""" from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) @@ -512,22 +484,18 @@ async def test_oauth_protected_resource_oauth_delegate_proxies_upstream_with_gat ) global_mcp_server_manager.registry[delegate_server.server_id] = delegate_server - _, mock_client = _make_upstream_metadata_client() + upstream_payload, mock_client = _make_upstream_metadata_client() try: - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): result = await _build_oauth_protected_resource_response( request=_make_request(), mcp_server_name="sample_docs", use_standard_pattern=True, ) - assert result["authorization_servers"] == [ - "https://okta.example.com/oauth2/default" - ] - assert result["resource"].endswith("/mcp/sample_docs") - assert result["resource"] != "https://upstream.example.com/mcp" + assert result == upstream_payload + assert result["authorization_servers"] == ["https://okta.example.com/oauth2/default"] + assert result["resource"] == "https://upstream.example.com/mcp" finally: global_mcp_server_manager.registry.clear() @@ -552,15 +520,11 @@ async def test_oauth_protected_resource_true_passthrough_returns_upstream_metada transport=MCPTransport.http, auth_type=MCPAuth.true_passthrough, ) - global_mcp_server_manager.registry[true_passthrough_server.server_id] = ( - true_passthrough_server - ) + global_mcp_server_manager.registry[true_passthrough_server.server_id] = true_passthrough_server upstream_payload, mock_client = _make_upstream_metadata_client() try: - with patch.object( - discoverable_endpoints, "get_async_httpx_client", return_value=mock_client - ): + with patch.object(discoverable_endpoints, "get_async_httpx_client", return_value=mock_client): result = await _build_oauth_protected_resource_response( request=_make_request(), mcp_server_name="sample_docs", From 732832d3426a1a0c29363e83d62be0546d338d8f Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 7 Jul 2026 19:46:23 -0700 Subject: [PATCH 004/101] fix(mcp): bind client-forwarded Authorization to a single upstream In a listing fan-out over a scope containing more than one server that consumes the caller's Authorization (true_passthrough, oauth_delegate, or the legacy delegate/passthrough shapes), the request-wide bearer is now withheld from the new modes instead of being replayed against every upstream (RFC 9700 cross-resource replay). Explicitly-addressed operations (tool call, get_prompt, read_resource, single-server routes) keep forwarding it. Multi-server aggregates use the per-server x-mcp-{alias}-authorization header instead: its value now feeds the passthrough resolver arm as the inbound token and wins over the request-wide header, binding one token to one server. --- .../mcp_server/mcp_server_manager.py | 51 ++ .../proxy/_experimental/mcp_server/server.py | 28 +- .../mcp_server/test_mcp_server.py | 583 +++++++----------- .../mcp_server/test_mcp_server_manager.py | 112 ++++ 4 files changed, 400 insertions(+), 374 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index fd978e5bfc9..acb7c80c5b7 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -368,6 +368,54 @@ def _take_forwarded_authorization( return value, _without_authorization(headers) +def _passthrough_token_from_mcp_auth_header( + mcp_auth_header: Optional[Union[str, dict[str, str]]], +) -> Optional[str]: + """The caller's per-server upstream credential for a passthrough-mode server, or None. + + Sourced from ``x-mcp-{alias}-authorization`` (string or per-header dict form) or the deprecated + global ``x-mcp-auth`` fallback. Per-server headers are the multi-server shape: they bind one + token to one server, so an aggregate scope with several passthrough-mode servers never replays + a single credential across upstreams. The value is forwarded verbatim, so it must be the full + header value (e.g. ``Bearer ``).""" + if isinstance(mcp_auth_header, str): + return mcp_auth_header or None + if isinstance(mcp_auth_header, dict): + return next((v for k, v in mcp_auth_header.items() if k.lower() == "authorization"), None) + return None + + +def _consumes_caller_authorization(server: MCPServer) -> bool: + """True when this server's egress forwards the caller's request-wide ``Authorization`` upstream: + the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated + interactive oauth2. An unstamped oauth2 row (flow column not yet backfilled) reads as a consumer, + which errs toward suppression — the fail-safe direction.""" + if server.is_true_passthrough or server.is_oauth_delegate or server.is_oauth_passthrough: + return True + return ( + server.auth_type == MCPAuth.oauth2 + and getattr(server, "delegate_auth_to_upstream", False) is True + and not server.has_client_credentials + ) + + +def _caller_authorization_fans_out( + server: MCPServer, + scope_servers: Optional[list[MCPServer]], +) -> bool: + """True when forwarding the caller's request-wide ``Authorization`` to ``server`` inside a + listing fan-out would replay one credential against multiple upstreams: another server in the + scope also consumes it (RFC 9700 cross-resource replay). ``scope_servers`` is None for + explicitly-addressed operations (tool call, get_prompt, read_resource, single-server routes), + where the client named the one target and the gateway is not choosing recipients.""" + if scope_servers is None: + return False + return any( + other is not None and other.server_id != server.server_id and _consumes_caller_authorization(other) + for other in scope_servers + ) + + def _extract_upstream_auth_failure( exc: BaseException, ) -> Optional[tuple[int, Optional[str]]]: @@ -2357,6 +2405,9 @@ class MCPServerManager: inbound_token = subject_token if isinstance(spec.config, PassthroughConfig): inbound_token, extra_headers = _take_forwarded_authorization(extra_headers) + per_server_token = _passthrough_token_from_mcp_auth_header(mcp_auth_header) + if per_server_token is not None: + inbound_token = per_server_token resolved_auth, extra_headers = await self._resolve_v2_auth( server=server, spec=spec, diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 7a854f698c9..4fda15f664e 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -331,6 +331,7 @@ if MCP_AVAILABLE: ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, + _caller_authorization_fans_out, _client_forwarded_authorization_headers, _should_strip_caller_authorization, _without_authorization, @@ -1529,8 +1530,16 @@ if MCP_AVAILABLE: oauth2_headers: Optional[Dict[str, str]], raw_headers: Optional[Dict[str, str]], user_api_key_auth: Optional[UserAPIKeyAuth] = None, + scope_servers: Optional[list[MCPServer]] = None, ) -> Tuple[Optional[Union[Dict[str, str], str]], Optional[Dict[str, str]]]: - """Build auth and extra headers for a server.""" + """Build auth and extra headers for a server. + + ``scope_servers`` is the full server list a fan-out handler iterates. Passing it lets the + client-forwarded token modes withhold the caller's request-wide ``Authorization`` when + another server in the scope would also receive it (``_caller_authorization_fans_out``); + explicitly-addressed operations leave it None. Per-server ``x-mcp-{alias}-authorization`` + headers are unaffected — they bind one token to one server and are the multi-server shape. + """ server_auth_header: Optional[Union[Dict[str, str], str]] = None if mcp_server_auth_headers: from litellm.proxy._experimental.mcp_server.utils import ( @@ -1563,12 +1572,13 @@ if MCP_AVAILABLE: ): extra_headers = _without_authorization(extra_headers) elif server.is_true_passthrough or server.is_oauth_delegate: - extra_headers = _client_forwarded_authorization_headers( - mcp_server=server, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - user_api_key_auth=user_api_key_auth, - ) + if not _caller_authorization_fans_out(server, scope_servers): + extra_headers = _client_forwarded_authorization_headers( + mcp_server=server, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) if server.extra_headers and raw_headers: if extra_headers is None: @@ -1793,6 +1803,7 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, ) # Prefer server-stored per-user OAuth when configured, so a stale @@ -1979,6 +1990,7 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, ) try: @@ -2031,6 +2043,7 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, ) try: @@ -2081,6 +2094,7 @@ if MCP_AVAILABLE: oauth2_headers=oauth2_headers, raw_headers=raw_headers, user_api_key_auth=user_api_key_auth, + scope_servers=allowed_mcp_servers, ) try: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index b24457deabd..71ece75b2fb 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -69,9 +69,7 @@ async def test_mcp_server_tool_call_body_contains_request_data(): # Mock the add_litellm_data_to_request function to capture the data captured_data = {} - async def mock_add_litellm_data_to_request( - data, request, user_api_key_dict, proxy_config - ): + async def mock_add_litellm_data_to_request(data, request, user_api_key_dict, proxy_config): captured_data.update(data) # Simulate the proxy_server_request creation captured_data["proxy_server_request"] = { @@ -355,6 +353,79 @@ def test_prepare_mcp_server_headers_m2m_skips_authorization_from_raw_extra_heade assert extra_headers.get("X-Custom") == "trace" +def _client_forwarded_mode_server(server_id: str, auth_type) -> MCPServer: + return MCPServer( + server_id=server_id, + name=server_id, + transport=MCPTransport.http, + auth_type=auth_type, + ) + + +def _prepare_headers_in_scope(server: MCPServer, scope_servers): + from litellm.proxy._experimental.mcp_server.server import ( + _prepare_mcp_server_headers, + ) + + return _prepare_mcp_server_headers( + server=server, + mcp_server_auth_headers=None, + mcp_auth_header=None, + oauth2_headers={"Authorization": "Bearer upstream-token"}, + raw_headers={ + "x-litellm-api-key": "Bearer sk-litellm-key", + "authorization": "Bearer upstream-token", + }, + user_api_key_auth=UserAPIKeyAuth(api_key="sk-litellm-key"), + scope_servers=scope_servers, + ) + + +def test_prepare_mcp_server_headers_withholds_global_authorization_when_scope_fans_out(): + """One caller bearer must not be replayed against multiple upstreams (RFC 9700 + cross-resource replay): in a fan-out scope with a second Authorization-consuming + server, the client-forwarded modes get no global Authorization.""" + delegate = _client_forwarded_mode_server("od-fanout", MCPAuth.oauth_delegate) + second_consumer = _client_forwarded_mode_server("tp-fanout", MCPAuth.true_passthrough) + + _, extra_headers = _prepare_headers_in_scope(delegate, [delegate, second_consumer]) + + assert not extra_headers or "authorization" not in {k.lower() for k in extra_headers} + + +def test_prepare_mcp_server_headers_forwards_global_authorization_to_sole_consumer(): + """Non-consuming servers (static api_key) in scope do not make the forward ambiguous.""" + delegate = _client_forwarded_mode_server("od-sole", MCPAuth.oauth_delegate) + static_server = MCPServer( + server_id="static-api-key", + name="static-api-key", + transport=MCPTransport.http, + auth_type=MCPAuth.api_key, + authentication_token="static-key", + ) + + _, extra_headers = _prepare_headers_in_scope(delegate, [delegate, static_server]) + + assert extra_headers == {"Authorization": "Bearer upstream-token"} + + +def test_prepare_mcp_server_headers_scope_counts_legacy_delegate_as_consumer(): + """Legacy upstream-delegated oauth2 servers still receive the caller's Authorization on the + v1 path, so their presence in scope must suppress the new modes' forward too.""" + delegate = _client_forwarded_mode_server("od-vs-legacy", MCPAuth.oauth_delegate) + legacy_delegate = MCPServer( + server_id="legacy-delegate", + name="legacy-delegate", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, + ) + + _, extra_headers = _prepare_headers_in_scope(delegate, [delegate, legacy_delegate]) + + assert not extra_headers or "authorization" not in {k.lower() for k in extra_headers} + + @pytest.mark.asyncio async def test_call_tool_m2m_skips_authorization_headers(): """M2M call_tool must not forward caller Authorization in oauth2/raw headers.""" @@ -382,9 +453,7 @@ async def test_call_tool_m2m_skips_authorization_headers(): mock_client = MagicMock() mock_client.call_tool = AsyncMock(return_value=MagicMock()) - with patch.object( - manager, "_create_mcp_client", new=AsyncMock(return_value=mock_client) - ) as create_client_mock: + with patch.object(manager, "_create_mcp_client", new=AsyncMock(return_value=mock_client)) as create_client_mock: await manager._call_regular_mcp_tool( mcp_server=server, original_tool_name="echo", @@ -879,9 +948,7 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): # Mock global_mcp_server_manager mock_manager = MagicMock() - mock_manager.get_allowed_mcp_servers = AsyncMock( - return_value=["working_server", "failing_server"] - ) + mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["working_server", "failing_server"]) mock_manager.get_mcp_server_by_id = lambda server_id: ( working_server if server_id == "working_server" else failing_server ) @@ -942,9 +1009,7 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): ) # Verify success logging - mock_logger.info.assert_any_call( - "Successfully fetched 1 tools total from all MCP servers" - ) + mock_logger.info.assert_any_call("Successfully fetched 1 tools total from all MCP servers") @pytest.mark.asyncio @@ -985,9 +1050,7 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): # Mock global_mcp_server_manager mock_manager = MagicMock() - mock_manager.get_allowed_mcp_servers = AsyncMock( - return_value=["failing_server1", "failing_server2"] - ) + mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["failing_server1", "failing_server2"]) mock_manager.get_mcp_server_by_id = lambda server_id: ( failing_server1 if server_id == "failing_server1" else failing_server2 ) @@ -1042,9 +1105,7 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): ) # Verify total logging - mock_logger.info.assert_any_call( - "Successfully fetched 0 tools total from all MCP servers" - ) + mock_logger.info.assert_any_call("Successfully fetched 0 tools total from all MCP servers") @pytest.mark.asyncio @@ -1069,9 +1130,7 @@ async def test_mcp_server_tool_call_body_with_none_arguments(): # Mock the add_litellm_data_to_request function to capture the data captured_data = {} - async def mock_add_litellm_data_to_request( - data, request, user_api_key_dict, proxy_config - ): + async def mock_add_litellm_data_to_request(data, request, user_api_key_dict, proxy_config): captured_data.update(data) captured_data["proxy_server_request"] = { "url": str(request.url), @@ -1176,31 +1235,29 @@ async def test_concurrent_initialize_session_managers(): results = await asyncio.gather(*tasks, return_exceptions=True) # All tasks should complete successfully (no exceptions) - assert all( - result == "success" for result in results - ), f"Some tasks failed: {results}" + assert all(result == "success" for result in results), f"Some tasks failed: {results}" # Each session manager.run() should only be called once due to the lock - assert ( - mock_stateless_run.call_count == 1 - ), f"Expected 1 call to session_manager_stateless.run(), got {mock_stateless_run.call_count}" - assert ( - mock_stateful_run.call_count == 1 - ), f"Expected 1 call to session_manager_stateful.run(), got {mock_stateful_run.call_count}" - assert ( - mock_sse_run.call_count == 1 - ), f"Expected 1 call to sse_session_manager.run(), got {mock_sse_run.call_count}" + assert mock_stateless_run.call_count == 1, ( + f"Expected 1 call to session_manager_stateless.run(), got {mock_stateless_run.call_count}" + ) + assert mock_stateful_run.call_count == 1, ( + f"Expected 1 call to session_manager_stateful.run(), got {mock_stateful_run.call_count}" + ) + assert mock_sse_run.call_count == 1, ( + f"Expected 1 call to sse_session_manager.run(), got {mock_sse_run.call_count}" + ) # The context managers should only be entered once each - assert ( - mock_cm_stateless.__aenter__.call_count == 1 - ), f"Expected 1 call to stateless __aenter__, got {mock_cm_stateless.__aenter__.call_count}" - assert ( - mock_cm_stateful.__aenter__.call_count == 1 - ), f"Expected 1 call to stateful __aenter__, got {mock_cm_stateful.__aenter__.call_count}" - assert ( - mock_cm_sse.__aenter__.call_count == 1 - ), f"Expected 1 call to sse __aenter__, got {mock_cm_sse.__aenter__.call_count}" + assert mock_cm_stateless.__aenter__.call_count == 1, ( + f"Expected 1 call to stateless __aenter__, got {mock_cm_stateless.__aenter__.call_count}" + ) + assert mock_cm_stateful.__aenter__.call_count == 1, ( + f"Expected 1 call to stateful __aenter__, got {mock_cm_stateful.__aenter__.call_count}" + ) + assert mock_cm_sse.__aenter__.call_count == 1, ( + f"Expected 1 call to sse __aenter__, got {mock_cm_sse.__aenter__.call_count}" + ) # State should be properly set assert mcp_server._SESSION_MANAGERS_INITIALIZED is True @@ -1343,16 +1400,12 @@ async def test_mcp_routing_initialize_to_stateful_no_session_to_stateless(): # initialize → stateful init_body = b'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05"}}' stateless_called, stateful_called = await make_request(init_body) - assert ( - stateful_called and not stateless_called - ), "initialize (no session) should route to stateful, not stateless" + assert stateful_called and not stateless_called, "initialize (no session) should route to stateful, not stateless" # tools/list → stateless tools_body = b'{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}' stateless_called, stateful_called = await make_request(tools_body) - assert ( - stateless_called and not stateful_called - ), "tools/list (no session) should route to stateless, not stateful" + assert stateless_called and not stateful_called, "tools/list (no session) should route to stateless, not stateful" @pytest.mark.asyncio @@ -1437,9 +1490,9 @@ async def test_mcp_routing_chunked_initialize_to_stateful(): ): await handle_streamable_http_mcp(scope, receive, send) - assert ( - stateful_called and not stateless_called - ), "chunked initialize (no session) should route to stateful, not stateless" + assert stateful_called and not stateless_called, ( + "chunked initialize (no session) should route to stateful, not stateless" + ) @pytest.mark.asyncio @@ -1468,10 +1521,7 @@ async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): messages = [ {"type": "http.request", "body": first_chunk, "more_body": True}, - *[ - {"type": "http.request", "body": chunk, "more_body": True} - for chunk in oversized_tail - ], + *[{"type": "http.request", "body": chunk, "more_body": True} for chunk in oversized_tail], {"type": "http.request", "body": b"", "more_body": False}, ] receive_calls = {"count": 0} @@ -1522,12 +1572,8 @@ async def test_mcp_routing_caps_body_peek_for_oversized_chunked_body(): "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), - patch.object( - session_manager_stateless, "handle_request", side_effect=stateless_handle - ), - patch.object( - session_manager_stateful, "handle_request", side_effect=stateful_handle - ), + patch.object(session_manager_stateless, "handle_request", side_effect=stateless_handle), + patch.object(session_manager_stateful, "handle_request", side_effect=stateful_handle), patch.object(session_manager_stateless, "_server_instances", {}), patch.object(session_manager_stateful, "_server_instances", {}), ): @@ -1578,9 +1624,7 @@ async def test_enforce_stateful_session_cap_evicts_oldest_idle_then_rejects(): patch.object(session_manager_stateful, "_server_instances", instances), patch.object(mcp_server, "_MAX_STATEFUL_SESSIONS_PER_OWNER", 3), patch.dict(mcp_server._stateful_session_owners, owners, clear=True), - patch.dict( - mcp_server._stateful_session_auth_context_last_seen, last_seen, clear=True - ), + patch.dict(mcp_server._stateful_session_auth_context_last_seen, last_seen, clear=True), patch.dict(mcp_server._stateful_session_auth_contexts, contexts, clear=True), patch.dict(mcp_server._stateful_session_active_request_counts, {}, clear=True), ): @@ -1593,9 +1637,7 @@ async def test_enforce_stateful_session_cap_evicts_oldest_idle_then_rejects(): # A different owner at the cap is unaffected by owner-A's sessions. terminated.clear() - allowed_other = await mcp_server._enforce_stateful_session_cap_for_owner( - "owner-B" - ) + allowed_other = await mcp_server._enforce_stateful_session_cap_for_owner("owner-B") assert allowed_other is True assert terminated == [] @@ -1614,9 +1656,7 @@ async def test_enforce_stateful_session_cap_evicts_oldest_idle_then_rejects(): {f"s{i}": float(i) for i in range(3)}, clear=True, ), - patch.dict( - mcp_server._stateful_session_active_request_counts, active, clear=True - ), + patch.dict(mcp_server._stateful_session_active_request_counts, active, clear=True), ): rejected = await mcp_server._enforce_stateful_session_cap_for_owner("owner-A") assert rejected is False @@ -1662,9 +1702,7 @@ async def test_mcp_routing_initialize_rejected_when_owner_at_session_cap(): (b"authorization", b"Bearer test-key"), ], } - receive = AsyncMock( - return_value={"type": "http.request", "body": init_body, "more_body": False} - ) + receive = AsyncMock(return_value={"type": "http.request", "body": init_body, "more_body": False}) send = AsyncMock() stateful_called = [] @@ -1685,9 +1723,7 @@ async def test_mcp_routing_initialize_rejected_when_owner_at_session_cap(): ), patch.object(mcp_server, "_owner_fingerprint_for", return_value="owner-X"), patch.object(mcp_server, "_MAX_STATEFUL_SESSIONS_PER_OWNER", cap), - patch.object( - session_manager_stateful, "handle_request", side_effect=stateful_handle - ), + patch.object(session_manager_stateful, "handle_request", side_effect=stateful_handle), patch.object(session_manager_stateful, "_server_instances", instances), patch.object(session_manager_stateless, "_server_instances", {}), patch.dict(mcp_server._stateful_session_owners, owners, clear=True), @@ -1696,18 +1732,14 @@ async def test_mcp_routing_initialize_rejected_when_owner_at_session_cap(): {f"s{i}": float(i) for i in range(cap)}, clear=True, ), - patch.dict( - mcp_server._stateful_session_active_request_counts, active, clear=True - ), + patch.dict(mcp_server._stateful_session_active_request_counts, active, clear=True), patch.dict(mcp_server._stateful_session_auth_contexts, contexts, clear=True), ): await handle_streamable_http_mcp(scope, receive, send) assert not stateful_called, "initialize at session cap must not reach the manager" start_messages = [ - call.args[0] - for call in send.call_args_list - if call.args and call.args[0].get("type") == "http.response.start" + call.args[0] for call in send.call_args_list if call.args and call.args[0].get("type") == "http.response.start" ] assert start_messages, "a response should have been sent" assert start_messages[0]["status"] == 429 @@ -1743,9 +1775,7 @@ async def test_stateful_mcp_requests_refresh_session_auth_context(): None, "1.1.1.1", ) - mcp_server._stateful_session_auth_contexts[session_id] = callback_context.run( - mcp_server.auth_context_var.get - ) + mcp_server._stateful_session_auth_contexts[session_id] = callback_context.run(mcp_server.auth_context_var.get) scope = { "type": "http", @@ -2006,24 +2036,14 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): ) await mcp_server._purge_expired_stateful_session_auth_contexts(now=now) assert new_session_id in mcp_server._stateful_session_auth_contexts - assert ( - mcp_server._stateful_session_auth_contexts[new_session_id] - is not existing_auth_user - ) - assert ( - mcp_server._stateful_session_auth_contexts[new_session_id].mcp_auth_header - == "new-mcp-auth" - ) + assert mcp_server._stateful_session_auth_contexts[new_session_id] is not existing_auth_user + assert mcp_server._stateful_session_auth_contexts[new_session_id].mcp_auth_header == "new-mcp-auth" async def stateless_handle(s, r, se): - raise AssertionError( - "initialize request with session should use stateful manager" - ) + raise AssertionError("initialize request with session should use stateful manager") try: - mcp_server._stateful_session_auth_contexts[existing_session_id] = ( - existing_auth_user - ) + mcp_server._stateful_session_auth_contexts[existing_session_id] = existing_auth_user mcp_server._stateful_session_auth_context_last_seen[existing_session_id] = 1.0 mcp_server._stateful_session_owners[existing_session_id] = owner_fingerprint @@ -2065,10 +2085,7 @@ async def test_initialize_request_with_existing_session_tracks_new_session(): assert stateful_called assert new_session_id not in mcp_server._stateful_session_active_request_counts assert new_session_id in mcp_server._stateful_session_auth_contexts - assert ( - mcp_server._stateful_session_auth_contexts[existing_session_id] - is existing_auth_user - ) + assert mcp_server._stateful_session_auth_contexts[existing_session_id] is existing_auth_user assert existing_auth_user.mcp_auth_header == "old-mcp-auth" assert existing_auth_user.mcp_servers == ["old-server"] finally: @@ -2191,15 +2208,11 @@ async def test_stateful_mcp_cleanup_loop_survives_purge_errors(): except ImportError: pytest.skip("MCP server not available") - purge = AsyncMock( - side_effect=[RuntimeError("terminate failed"), asyncio.CancelledError()] - ) + purge = AsyncMock(side_effect=[RuntimeError("terminate failed"), asyncio.CancelledError()]) with ( patch.object(mcp_server.asyncio, "sleep", AsyncMock(return_value=None)), - patch.object( - mcp_server, "_purge_expired_stateful_session_auth_contexts", purge - ), + patch.object(mcp_server, "_purge_expired_stateful_session_auth_contexts", purge), ): with pytest.raises(asyncio.CancelledError): await mcp_server._cleanup_expired_stateful_session_auth_contexts() @@ -2289,9 +2302,7 @@ async def test_stateful_mcp_session_owner_mismatch_returns_403(): intruder_auth = UserAPIKeyAuth(api_key="intruder-key", user_id="intruder") mcp_server._stateful_session_auth_contexts[session_id] = MagicMock() - mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for( - owner_auth - ) + mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for(owner_auth) scope = { "type": "http", @@ -2341,9 +2352,7 @@ async def test_stateful_mcp_session_owner_mismatch_returns_403(): await handle_streamable_http_mcp(scope, receive, capture_send) handle_request_mock.assert_not_awaited() - statuses = [ - m["status"] for m in sent_messages if m.get("type") == "http.response.start" - ] + statuses = [m["status"] for m in sent_messages if m.get("type") == "http.response.start"] assert statuses == [403] mcp_server._stateful_session_auth_contexts.pop(session_id, None) @@ -2368,12 +2377,10 @@ async def test_stateful_mcp_session_serializes_concurrent_requests(): session_id = "serialized-session-1" owner_auth = UserAPIKeyAuth(api_key="owner-key", user_id="owner") - mcp_server._stateful_session_auth_contexts[session_id] = ( - mcp_server.MCPAuthenticatedUser(user_api_key_auth=owner_auth) - ) - mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for( - owner_auth + mcp_server._stateful_session_auth_contexts[session_id] = mcp_server.MCPAuthenticatedUser( + user_api_key_auth=owner_auth ) + mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for(owner_auth) inside = 0 max_inside = 0 @@ -2414,9 +2421,7 @@ async def test_stateful_mcp_session_serializes_concurrent_requests(): "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), - patch.object( - session_manager_stateful, "handle_request", side_effect=slow_handle - ), + patch.object(session_manager_stateful, "handle_request", side_effect=slow_handle), patch.object( session_manager_stateful, "_server_instances", @@ -2432,9 +2437,7 @@ async def test_stateful_mcp_session_serializes_concurrent_requests(): mcp_server._stateful_session_owners.pop(session_id, None) mcp_server._stateful_session_locks.pop(session_id, None) - assert ( - max_inside == 1 - ), "concurrent requests on same stateful session must be serialized" + assert max_inside == 1, "concurrent requests on same stateful session must be serialized" @pytest.mark.asyncio @@ -2486,9 +2489,7 @@ async def test_stateful_mcp_lock_does_not_leak_when_auth_context_missing(): "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), - patch.object( - session_manager_stateful, "handle_request", side_effect=handle - ), + patch.object(session_manager_stateful, "handle_request", side_effect=handle), patch.object( session_manager_stateful, "_server_instances", @@ -2498,9 +2499,9 @@ async def test_stateful_mcp_lock_does_not_leak_when_auth_context_missing(): assert session_id not in mcp_server._stateful_session_auth_contexts await handle_streamable_http_mcp(scope, receive, AsyncMock()) - assert ( - session_id not in mcp_server._stateful_session_locks - ), "lock entry must be cleaned up for untracked stateful session" + assert session_id not in mcp_server._stateful_session_locks, ( + "lock entry must be cleaned up for untracked stateful session" + ) finally: mcp_server._stateful_session_auth_contexts.pop(session_id, None) mcp_server._stateful_session_owners.pop(session_id, None) @@ -2526,12 +2527,10 @@ async def test_stateful_mcp_get_stream_does_not_block_post(): session_id = "stream-session-1" owner_auth = UserAPIKeyAuth(api_key="owner-key", user_id="owner") - mcp_server._stateful_session_auth_contexts[session_id] = ( - mcp_server.MCPAuthenticatedUser(user_api_key_auth=owner_auth) - ) - mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for( - owner_auth + mcp_server._stateful_session_auth_contexts[session_id] = mcp_server.MCPAuthenticatedUser( + user_api_key_auth=owner_auth ) + mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for(owner_auth) stream_release = asyncio.Event() post_finished = asyncio.Event() @@ -2569,9 +2568,7 @@ async def test_stateful_mcp_get_stream_does_not_block_post(): "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), - patch.object( - session_manager_stateful, "handle_request", side_effect=handle - ), + patch.object(session_manager_stateful, "handle_request", side_effect=handle), patch.object( session_manager_stateful, "_server_instances", @@ -2615,10 +2612,7 @@ def test_jsonrpc_text_has_top_level_method_ignores_nested_method(): assert _jsonrpc_text_has_top_level_method(reordered) is True # response whose result nests a "method" key (and arrays of them) - response = ( - '{"jsonrpc":"2.0","id":1,"result":{"toolResult":{"method":"GET"},' - '"steps":[{"method":"x"}]}}' - ) + response = '{"jsonrpc":"2.0","id":1,"result":{"toolResult":{"method":"GET"},"steps":[{"method":"x"}]}}' assert _jsonrpc_text_has_top_level_method(response) is False # truncated response: result value never closes, no top-level method seen @@ -2643,12 +2637,10 @@ async def test_truncated_jsonrpc_response_with_nested_method_skips_lock(): session_id = "nested-method-response-session" owner_auth = UserAPIKeyAuth(api_key="owner-key", user_id="owner") - mcp_server._stateful_session_auth_contexts[session_id] = ( - mcp_server.MCPAuthenticatedUser(user_api_key_auth=owner_auth) - ) - mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for( - owner_auth + mcp_server._stateful_session_auth_contexts[session_id] = mcp_server.MCPAuthenticatedUser( + user_api_key_auth=owner_auth ) + mcp_server._stateful_session_owners[session_id] = mcp_server._owner_fingerprint_for(owner_auth) gate = asyncio.Event() request_in_handle = asyncio.Event() @@ -2685,8 +2677,7 @@ async def test_truncated_jsonrpc_response_with_nested_method_skips_lock(): # parsed, with a nested "method" key in the first bytes to trip a flat # substring heuristic. response_body = ( - '{"jsonrpc":"2.0","id":99,"result":{"toolResult":' - '{"method":"GET","payload":"' + ("x" * 5000) + '"}}}' + '{"jsonrpc":"2.0","id":99,"result":{"toolResult":{"method":"GET","payload":"' + ("x" * 5000) + '"}}}' ).encode() try: @@ -2700,9 +2691,7 @@ async def test_truncated_jsonrpc_response_with_nested_method_skips_lock(): "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, ), - patch.object( - session_manager_stateful, "handle_request", side_effect=handle - ), + patch.object(session_manager_stateful, "handle_request", side_effect=handle), patch.object( session_manager_stateful, "_server_instances", @@ -2774,13 +2763,9 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name(): mock_get_tools_spy = AsyncMock(return_value=[]) # Mock the function that checks DB for an access group named "custom_solutions" - mock_db_lookup = AsyncMock( - return_value=[specific_server.server_id, other_server.server_id] - ) + mock_db_lookup = AsyncMock(return_value=[specific_server.server_id, other_server.server_id]) - mock_get_allowed = AsyncMock( - return_value=[specific_server.server_id, other_server.server_id] - ) + mock_get_allowed = AsyncMock(return_value=[specific_server.server_id, other_server.server_id]) with ( patch( @@ -2805,14 +2790,12 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name(): ) # Get the list of actual server objects that the orchestrator tried to contact - called_servers = [ - call.kwargs["server"] for call in mock_get_tools_spy.call_args_list - ] + called_servers = [call.kwargs["server"] for call in mock_get_tools_spy.call_args_list] assert len(called_servers) == 1, "Should have resolved to exactly one server." - assert ( - called_servers[0].server_id == specific_server.server_id - ), "Should have contacted the specific server alias, not the group." + assert called_servers[0].server_id == specific_server.server_id, ( + "Should have contacted the specific server alias, not the group." + ) @pytest.mark.asyncio @@ -2926,9 +2909,7 @@ async def test_oauth2_caller_headers_not_forwarded_for_migrated_server(): ) # Verify that _create_mcp_client was called - assert ( - mock_create_client.call_count == 1 - ), "Expected _create_mcp_client to be called once" + assert mock_create_client.call_count == 1, "Expected _create_mcp_client to be called once" # Verify the server passed to _create_mcp_client is the OAuth2 server assert captured_client_args["server"].server_id == oauth2_server.server_id @@ -2938,9 +2919,9 @@ async def test_oauth2_caller_headers_not_forwarded_for_migrated_server(): # oauth2 Authorization upstream. The v2 resolver injects the stored per-user token, # so a caller-supplied bearer cannot override another user's stored credential. extra_headers = captured_client_args["extra_headers"] - assert extra_headers is None or "Authorization" not in { - k.lower() for k in extra_headers - }, f"Caller Authorization must not be forwarded, got {extra_headers}" + assert extra_headers is None or "Authorization" not in {k.lower() for k in extra_headers}, ( + f"Caller Authorization must not be forwarded, got {extra_headers}" + ) @pytest.mark.asyncio @@ -3049,12 +3030,8 @@ async def test_list_tools_multiple_servers_prefixed_names(): # Mock manager mock_manager = MagicMock() - mock_manager.get_allowed_mcp_servers = AsyncMock( - return_value=["server1", "server2"] - ) - mock_manager.get_mcp_server_by_id = lambda server_id: ( - server1 if server_id == "server1" else server2 - ) + mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["server1", "server2"]) + mock_manager.get_mcp_server_by_id = lambda server_id: server1 if server_id == "server1" else server2 # Mock filter_server_ids_by_ip to return server_ids unchanged (no IP filtering) mock_manager.filter_server_ids_by_ip_with_info = lambda server_ids, client_ip: ( server_ids, @@ -3653,9 +3630,7 @@ async def test_list_tools_strips_prefix_when_matching_permissions(): tool1.inputSchema = {} tool2 = MagicMock() - tool2.name = ( - "GITMCP-search_litellm_documentation" # Prefixed, not in allowed list - ) + tool2.name = "GITMCP-search_litellm_documentation" # Prefixed, not in allowed list tool2.description = "Search docs" tool2.inputSchema = {} @@ -3876,17 +3851,13 @@ class TestMCPServerManagerReload: db_row = _make_db_mcp_server("server-1", timestamp) mock_prisma = MagicMock() - mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock( - return_value=[db_row] - ) + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[db_row]) with ( patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", return_value=mock_prisma, ), - patch.object( - manager, "build_mcp_server_from_table", AsyncMock() - ) as mock_build, + patch.object(manager, "build_mcp_server_from_table", AsyncMock()) as mock_build, ): await manager.reload_servers_from_database() @@ -3922,9 +3893,7 @@ class TestMCPServerManagerReload: ) mock_prisma = MagicMock() - mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock( - return_value=[db_row] - ) + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[db_row]) with ( patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", @@ -4049,9 +4018,7 @@ class TestMCPServerManagerReload: raise RuntimeError("blocked address") mock_prisma = MagicMock() - mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock( - return_value=[healthy_row, bad_openapi_row] - ) + mock_prisma.db.litellm_mcpservertable.find_many = AsyncMock(return_value=[healthy_row, bad_openapi_row]) with ( patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", @@ -4147,10 +4114,7 @@ async def test_call_mcp_tool_logs_failure_via_post_call_failure_hook(): ) proxy_logging_mock.post_call_failure_hook.assert_awaited_once() - assert ( - proxy_logging_mock.post_call_failure_hook.await_args.kwargs.get("route") - == "/mcp/call_tool" - ) + assert proxy_logging_mock.post_call_failure_hook.await_args.kwargs.get("route") == "/mcp/call_tool" @pytest.mark.asyncio @@ -4232,9 +4196,7 @@ async def test_get_tools_from_mcp_servers_logs_list_tools_to_spendlogs_when_enab assert tools == [tool_1] dummy_logging_obj.async_success_handler.assert_awaited_once() - assert dummy_logging_obj.async_success_handler.await_args.kwargs["result"] == [ - tool_1.model_dump(mode="json") - ] + assert dummy_logging_obj.async_success_handler.await_args.kwargs["result"] == [tool_1.model_dump(mode="json")] assert function_setup_kwargs["metadata"]["tags"] == ["team-a"] spend_meta = dummy_logging_obj.model_call_details["metadata"]["spend_logs_metadata"] @@ -4580,9 +4542,7 @@ async def test_get_tools_from_mcp_servers_injects_stored_oauth2_token(): oauth2_server.extra_headers = None # Simulate the DB returning a valid credential for this user+server - prefetched_creds = { - SERVER_ID: {"access_token": STORED_TOKEN, "server_id": SERVER_ID} - } + prefetched_creds = {SERVER_ID: {"access_token": STORED_TOKEN, "server_id": SERVER_ID}} tool_1 = MagicMock() tool_1.name = "atlassian_test-search" @@ -4689,16 +4649,12 @@ class TestMergeGatewayInitializeInstructions: global_mcp_server_manager, ) - global_mcp_server_manager._upstream_initialize_instructions_by_server_id[ - "s1" - ] = "upstream" + global_mcp_server_manager._upstream_initialize_instructions_by_server_id["s1"] = "upstream" try: s = _make_instruction_server(instructions="yaml wins") assert self._merge([s]) == "yaml wins" finally: - global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop( - "s1", None - ) + global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop("s1", None) def test_upstream_cache_used_when_no_yaml(self): """Upstream cached instructions are used when no YAML override is set.""" @@ -4706,16 +4662,12 @@ class TestMergeGatewayInitializeInstructions: global_mcp_server_manager, ) - global_mcp_server_manager._upstream_initialize_instructions_by_server_id[ - "s1" - ] = "from upstream" + global_mcp_server_manager._upstream_initialize_instructions_by_server_id["s1"] = "from upstream" try: s = _make_instruction_server(instructions=None) assert self._merge([s]) == "from upstream" finally: - global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop( - "s1", None - ) + global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop("s1", None) def test_spec_path_servers_skipped(self): """OpenAPI (spec_path) servers do not contribute instructions.""" @@ -4729,12 +4681,8 @@ class TestMergeGatewayInitializeInstructions: def test_multiple_servers_merged_with_labels(self): """Multiple servers get label-prefixed and separator-joined.""" - s1 = _make_instruction_server( - server_id="a", name="a", alias="Alpha", instructions="instr A" - ) - s2 = _make_instruction_server( - server_id="b", name="b", alias="Beta", instructions="instr B" - ) + s1 = _make_instruction_server(server_id="a", name="a", alias="Alpha", instructions="instr A") + s2 = _make_instruction_server(server_id="b", name="b", alias="Beta", instructions="instr B") result = self._merge([s1, s2]) assert result is not None assert "[Alpha]" in result and "[Beta]" in result @@ -4754,25 +4702,17 @@ class TestMergeGatewayInitializeInstructions: global_mcp_server_manager, ) - global_mcp_server_manager._upstream_initialize_instructions_by_server_id[ - "c" - ] = "cached C" + global_mcp_server_manager._upstream_initialize_instructions_by_server_id["c"] = "cached C" try: - s_yaml = _make_instruction_server( - server_id="a", name="a", alias="A", instructions="yaml A" - ) - s_spec = _make_instruction_server( - server_id="b", name="b", alias="B", spec_path="/spec.json", url=None - ) + s_yaml = _make_instruction_server(server_id="a", name="a", alias="A", instructions="yaml A") + s_spec = _make_instruction_server(server_id="b", name="b", alias="B", spec_path="/spec.json", url=None) s_cached = _make_instruction_server(server_id="c", name="c", alias="C") result = self._merge([s_yaml, s_spec, s_cached]) assert "yaml A" in result assert "cached C" in result assert "[B]" not in result finally: - global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop( - "c", None - ) + global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop("c", None) class TestEnsureUpstreamInitializeInstructionsCached: @@ -4784,15 +4724,9 @@ class TestEnsureUpstreamInitializeInstructionsCached: global_mcp_server_manager, ) - server = _make_instruction_server( - server_id="yaml-only", instructions="from yaml" - ) - with patch.object( - global_mcp_server_manager, "_create_mcp_client", AsyncMock() - ) as mock_create: - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) + server = _make_instruction_server(server_id="yaml-only", instructions="from yaml") + with patch.object(global_mcp_server_manager, "_create_mcp_client", AsyncMock()) as mock_create: + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) mock_create.assert_not_awaited() @pytest.mark.asyncio @@ -4804,21 +4738,13 @@ class TestEnsureUpstreamInitializeInstructionsCached: ) server = _make_instruction_server(server_id="cached-only", instructions=None) - global_mcp_server_manager._upstream_initialize_instructions_by_server_id[ - "cached-only" - ] = "warm" + global_mcp_server_manager._upstream_initialize_instructions_by_server_id["cached-only"] = "warm" try: - with patch.object( - global_mcp_server_manager, "_create_mcp_client", AsyncMock() - ) as mock_create: - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) + with patch.object(global_mcp_server_manager, "_create_mcp_client", AsyncMock()) as mock_create: + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) mock_create.assert_not_awaited() finally: - global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop( - "cached-only", None - ) + global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop("cached-only", None) @pytest.mark.asyncio async def test_skips_when_spec_path_set(self): @@ -4828,15 +4754,9 @@ class TestEnsureUpstreamInitializeInstructionsCached: global_mcp_server_manager, ) - server = _make_instruction_server( - server_id="openapi-spec", spec_path="/openapi.json", url=None - ) - with patch.object( - global_mcp_server_manager, "_create_mcp_client", AsyncMock() - ) as mock_create: - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) + server = _make_instruction_server(server_id="openapi-spec", spec_path="/openapi.json", url=None) + with patch.object(global_mcp_server_manager, "_create_mcp_client", AsyncMock()) as mock_create: + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) mock_create.assert_not_awaited() @pytest.mark.asyncio @@ -4858,22 +4778,14 @@ class TestEnsureUpstreamInitializeInstructionsCached: AsyncMock(return_value=fake_client), ): try: - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) assert ( - global_mcp_server_manager._upstream_initialize_instructions_by_server_id[ - "cold-server" - ] + global_mcp_server_manager._upstream_initialize_instructions_by_server_id["cold-server"] == "upstream says hi" ) finally: - global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop( - "cold-server", None - ) - global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop( - "cold-server", None - ) + global_mcp_server_manager._upstream_initialize_instructions_by_server_id.pop("cold-server", None) + global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop("cold-server", None) @pytest.mark.asyncio async def test_cooldown_after_empty_upstream_response(self): @@ -4892,27 +4804,13 @@ class TestEnsureUpstreamInitializeInstructionsCached: create = AsyncMock(return_value=fake_client) with patch.object(global_mcp_server_manager, "_create_mcp_client", create): try: - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) - assert ( - create.await_count == 1 - ), "Second probe within cooldown must not reconnect to upstream" - assert ( - "empty-server" - not in global_mcp_server_manager._upstream_initialize_instructions_by_server_id - ) - assert ( - "empty-server" - in global_mcp_server_manager._upstream_initialize_instructions_probed_at - ) + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) + assert create.await_count == 1, "Second probe within cooldown must not reconnect to upstream" + assert "empty-server" not in global_mcp_server_manager._upstream_initialize_instructions_by_server_id + assert "empty-server" in global_mcp_server_manager._upstream_initialize_instructions_probed_at finally: - global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop( - "empty-server", None - ) + global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop("empty-server", None) @pytest.mark.asyncio async def test_cooldown_after_upstream_failure(self): @@ -4925,35 +4823,19 @@ class TestEnsureUpstreamInitializeInstructionsCached: server = _make_instruction_server(server_id="boom-server", instructions=None) fake_client = MagicMock() - fake_client.run_with_session = AsyncMock( - side_effect=RuntimeError("upstream down") - ) + fake_client.run_with_session = AsyncMock(side_effect=RuntimeError("upstream down")) fake_client._last_initialize_instructions = None create = AsyncMock(return_value=fake_client) with patch.object(global_mcp_server_manager, "_create_mcp_client", create): try: - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) - await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached( - server - ) - assert ( - create.await_count == 1 - ), "Second probe within cooldown must not reconnect after failure" - assert ( - "boom-server" - not in global_mcp_server_manager._upstream_initialize_instructions_by_server_id - ) - assert ( - "boom-server" - in global_mcp_server_manager._upstream_initialize_instructions_probed_at - ) + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) + await global_mcp_server_manager._ensure_upstream_initialize_instructions_cached(server) + assert create.await_count == 1, "Second probe within cooldown must not reconnect after failure" + assert "boom-server" not in global_mcp_server_manager._upstream_initialize_instructions_by_server_id + assert "boom-server" in global_mcp_server_manager._upstream_initialize_instructions_probed_at finally: - global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop( - "boom-server", None - ) + global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop("boom-server", None) @pytest.mark.asyncio async def test_reload_resets_probe_cooldown(self): @@ -4962,19 +4844,12 @@ class TestEnsureUpstreamInitializeInstructionsCached: global_mcp_server_manager, ) - global_mcp_server_manager._upstream_initialize_instructions_probed_at[ - "reload-target" - ] = 1.0 + global_mcp_server_manager._upstream_initialize_instructions_probed_at["reload-target"] = 1.0 try: await global_mcp_server_manager.load_servers_from_config({}) - assert ( - "reload-target" - not in global_mcp_server_manager._upstream_initialize_instructions_probed_at - ) + assert "reload-target" not in global_mcp_server_manager._upstream_initialize_instructions_probed_at finally: - global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop( - "reload-target", None - ) + global_mcp_server_manager._upstream_initialize_instructions_probed_at.pop("reload-target", None) class TestGatewayCreateInitializationOptions: @@ -5040,9 +4915,7 @@ class TestGatewayCreateInitializationOptions: ): assert server.create_initialization_options().server_name == "grafana" - assert ( - server.create_initialization_options().server_name == "litellm-mcp-server" - ) + assert server.create_initialization_options().server_name == "litellm-mcp-server" @pytest.mark.asyncio async def test_sse_handler_scopes_server_name_from_single_server_path(self): @@ -5119,9 +4992,7 @@ class TestGatewayCreateInitializationOptions: await handle_sse_mcp(scope, AsyncMock(), AsyncMock()) assert captured["server_name"] == "grafana" - assert ( - server.create_initialization_options().server_name == "litellm-mcp-server" - ) + assert server.create_initialization_options().server_name == "litellm-mcp-server" def test_contextvar_set_injects_instructions(self): """When ContextVar has a value, it appears in InitializationOptions.""" @@ -5239,12 +5110,8 @@ async def test_list_tools_with_legacy_db_m2m_server_resolves_oauth2_flow(): ): mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["legacy-m2m-id"]) mock_manager.get_mcp_server_by_id = MagicMock(return_value=legacy_server) - mock_manager.filter_server_ids_by_ip_with_info = MagicMock( - return_value=(["legacy-m2m-id"], 0) - ) - mock_manager._get_tools_from_server = AsyncMock( - side_effect=capture_extra_headers - ) + mock_manager.filter_server_ids_by_ip_with_info = MagicMock(return_value=(["legacy-m2m-id"], 0)) + mock_manager._get_tools_from_server = AsyncMock(side_effect=capture_extra_headers) tools = await _get_tools_from_mcp_servers( user_api_key_auth=user_auth, @@ -5336,10 +5203,8 @@ async def test_call_tool_empty_extra_headers_returns_none(): pass # We only care about the captured headers # With P2 fix: extra_headers should be None (not {}) when all headers filtered - assert ( - captured_extra_headers is None - ), "P2 API consistency issue: expected None for empty extra_headers, got: " + str( - captured_extra_headers + assert captured_extra_headers is None, ( + "P2 API consistency issue: expected None for empty extra_headers, got: " + str(captured_extra_headers) ) @@ -5364,9 +5229,7 @@ async def test_probe_upstream_auth_returns_upstream_status(): "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", return_value=mock_client, ): - status, www_auth = await _probe_upstream_auth( - "http://upstream/mcp", "Bearer some-token" - ) + status, www_auth = await _probe_upstream_auth("http://upstream/mcp", "Bearer some-token") assert status == 401 assert www_auth == 'Bearer realm="test"' @@ -5393,9 +5256,7 @@ async def test_probe_upstream_auth_surfaces_httpx_status_error(): mock_response.status_code = 401 mock_response.headers = {"www-authenticate": 'Bearer realm="test"'} request = httpx.Request("POST", "http://upstream/mcp") - error = httpx.HTTPStatusError( - message="401 Unauthorized", request=request, response=mock_response - ) + error = httpx.HTTPStatusError(message="401 Unauthorized", request=request, response=mock_response) mock_client = MagicMock() mock_client.post = AsyncMock(side_effect=error) @@ -5404,9 +5265,7 @@ async def test_probe_upstream_auth_surfaces_httpx_status_error(): "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", return_value=mock_client, ): - status, www_auth = await _probe_upstream_auth( - "http://upstream/mcp", "Bearer some-token" - ) + status, www_auth = await _probe_upstream_auth("http://upstream/mcp", "Bearer some-token") assert status == 401 assert www_auth == 'Bearer realm="test"' @@ -5424,9 +5283,7 @@ async def test_probe_upstream_auth_fails_open_on_network_error(): "litellm.proxy._experimental.mcp_server.server.get_async_httpx_client", return_value=mock_client, ): - status, www_auth = await _probe_upstream_auth( - "http://upstream/mcp", "Bearer some-token" - ) + status, www_auth = await _probe_upstream_auth("http://upstream/mcp", "Bearer some-token") assert status == 200 assert www_auth is None @@ -6349,9 +6206,7 @@ class TestProxyExceptionToHttpException: from litellm.proxy._types import ProxyException http_exc = _proxy_exception_to_http_exception( - ProxyException( - message="Forbidden", type="auth_error", param="key", code=403 - ) + ProxyException(message="Forbidden", type="auth_error", param="key", code=403) ) assert http_exc.status_code == 403 @@ -6415,10 +6270,7 @@ class TestStreamableHttpAuthErrorMapping: assert exc_info.value.status_code == 401 assert exc_info.value.headers["WWW-Authenticate"] == "Bearer" # Must not have emitted a 500 body via the generic catch-all. - assert not any( - m.get("type") == "http.response.start" and m.get("status") == 500 - for m in sent - ) + assert not any(m.get("type") == "http.response.start" and m.get("status") == 500 for m in sent) @pytest.mark.asyncio async def test_sse_propagates_proxy_exception_as_401(self): @@ -6458,10 +6310,7 @@ class TestStreamableHttpAuthErrorMapping: assert exc_info.value.status_code == 401 assert exc_info.value.headers["WWW-Authenticate"] == "Bearer" - assert not any( - m.get("type") == "http.response.start" and m.get("status") == 500 - for m in sent - ) + assert not any(m.get("type") == "http.response.start" and m.get("status") == 500 for m in sent) class TestMCPMetaTraceCarrier: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index a7a0379c7ad..dc1a950f706 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -1490,6 +1490,118 @@ class TestMCPServerManager: assert emitted.headers["Authorization"] == "Bearer upstream-token" assert not kwargs["extra_headers"] or "authorization" not in {k.lower() for k in kwargs["extra_headers"]} + @staticmethod + def _emitted_authorization(mock_client_cls) -> str: + kwargs = mock_client_cls.call_args.kwargs + emitted = httpx.Request("GET", "https://example.com/mcp") + flow = kwargs["resolved_auth"].auth_flow(emitted) + next(flow) + flow.close() + return emitted.headers["Authorization"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "per_server_header", + ["Bearer per-server-token", {"Authorization": "Bearer per-server-token"}], + ) + async def test_create_mcp_client_passthrough_prefers_per_server_token(self, per_server_header): + """A per-server x-mcp-{alias}-authorization value is the explicit one-token-one-server + binding, so it must win over the request-wide Authorization and reach the upstream + verbatim through the passthrough arm.""" + manager = MCPServerManager() + server = MCPServer( + server_id="tp-per-server", + name="tp", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.true_passthrough, + ) + with ( + patch( + "litellm.proxy._experimental.mcp_server.mcp_server_manager.resolve_mcp_auth", + new_callable=AsyncMock, + ) as mock_resolve, + patch("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCPClient") as mock_client_cls, + ): + await manager._create_mcp_client( + server=server, + mcp_auth_header=per_server_header, + extra_headers={"Authorization": "Bearer global-token"}, + ) + mock_resolve.assert_not_awaited() + assert self._emitted_authorization(mock_client_cls) == "Bearer per-server-token" + kwargs = mock_client_cls.call_args.kwargs + assert not kwargs["extra_headers"] or "authorization" not in {k.lower() for k in kwargs["extra_headers"]} + + def test_consumes_caller_authorization_per_mode(self): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _consumes_caller_authorization, + ) + + def build(**kwargs) -> MCPServer: + return MCPServer( + server_id="s", + name="s", + url="https://example.com", + transport=MCPTransport.http, + **kwargs, + ) + + assert _consumes_caller_authorization(build(auth_type=MCPAuth.true_passthrough)) is True + assert _consumes_caller_authorization(build(auth_type=MCPAuth.oauth_delegate)) is True + assert ( + _consumes_caller_authorization( + build(auth_type=MCPAuth.none, extra_headers=["Authorization"], oauth_passthrough=True) + ) + is True + ) + assert _consumes_caller_authorization(build(auth_type=MCPAuth.oauth2, delegate_auth_to_upstream=True)) is True + assert _consumes_caller_authorization(build(auth_type=MCPAuth.api_key, authentication_token="x")) is False + assert ( + _consumes_caller_authorization( + build( + auth_type=MCPAuth.oauth2, + delegate_auth_to_upstream=True, + oauth2_flow="client_credentials", + token_url="https://idp/token", + ) + ) + is False + ) + + def test_caller_authorization_fans_out_only_with_second_consumer(self): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _caller_authorization_fans_out, + ) + + delegate = MCPServer( + server_id="od", + name="od", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + ) + second = MCPServer( + server_id="tp", + name="tp", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.true_passthrough, + ) + static_server = MCPServer( + server_id="static", + name="static", + url="https://example.com", + transport=MCPTransport.http, + auth_type=MCPAuth.api_key, + authentication_token="x", + ) + + assert _caller_authorization_fans_out(delegate, None) is False + assert _caller_authorization_fans_out(delegate, [delegate]) is False + assert _caller_authorization_fans_out(delegate, [delegate, static_server]) is False + assert _caller_authorization_fans_out(delegate, [delegate, second]) is True + @pytest.mark.asyncio async def test_get_prompts_from_server_success(self): """Ensure prompts are fetched and prefixed when requested.""" From 34db5f4813ab3449ef489a17b9d7b3da9d7c6635 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Wed, 8 Jul 2026 16:26:47 +1000 Subject: [PATCH 005/101] feat(ui): add start time sort toggle to session logs sidebar --- .../LogDetailsDrawer.test.tsx | 105 ++++++++++++++++++ .../LogDetailsDrawer/LogDetailsDrawer.tsx | 39 ++++--- .../view_logs/LogDetailsDrawer/utils.test.ts | 30 +++++ .../view_logs/LogDetailsDrawer/utils.ts | 21 ++++ 4 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx new file mode 100644 index 00000000000..db0d168fcac --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx @@ -0,0 +1,105 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import { LogDetailsDrawer } from "./LogDetailsDrawer"; +import { sessionSpendLogsCall } from "../../networking"; +import { LogEntry } from "../columns"; + +vi.mock("../../networking", () => ({ + sessionSpendLogsCall: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/logDetails/useLogDetails", () => ({ + useLogDetails: () => ({ data: null, isLoading: false }), +})); + +vi.mock("./LogDetailContent", () => ({ + LogDetailContent: () => null, + GuardrailJumpLink: () => null, +})); + +vi.mock("./DrawerHeader", () => ({ + DrawerHeader: () => null, +})); + +const makeLog = (overrides: Partial): LogEntry => ({ + request_id: "req", + api_key: "", + team_id: "", + model: "", + model_id: "", + call_type: "acompletion", + spend: 0, + total_tokens: 0, + prompt_tokens: 0, + completion_tokens: 0, + startTime: "2026-07-08T10:00:00.000Z", + endTime: "2026-07-08T10:00:01.000Z", + cache_hit: "false", + messages: [], + response: {}, + ...overrides, +}); + +const sessionLogs = [ + makeLog({ + request_id: "llm-early", + model: "llm-early", + startTime: "2026-07-08T10:00:00.000Z", + endTime: "2026-07-08T10:00:02.000Z", + }), + makeLog({ + request_id: "mcp-early", + model: "tool-early", + call_type: "call_mcp_tool", + startTime: "2026-07-08T10:00:01.000Z", + endTime: "2026-07-08T10:00:01.500Z", + }), + makeLog({ + request_id: "llm-late", + model: "llm-late", + startTime: "2026-07-08T10:00:02.000Z", + endTime: "2026-07-08T10:00:04.000Z", + }), + makeLog({ + request_id: "mcp-late", + model: "tool-late", + call_type: "call_mcp_tool", + startTime: "2026-07-08T10:00:03.000Z", + endTime: "2026-07-08T10:00:03.500Z", + }), +]; + +const renderSessionDrawer = () => { + vi.mocked(sessionSpendLogsCall).mockResolvedValue({ data: sessionLogs, total: 4, total_pages: 1 }); + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render( + + {}} logEntry={null} sessionId="session-1" accessToken="token" /> + , + ); +}; + +const sidebarEventNames = () => + screen.queryAllByText(/^(llm-early|llm-late|tool-early|tool-late)$/).map((el) => el.textContent); + +describe("LogDetailsDrawer session sidebar sorting", () => { + it("defaults to grouped order: LLM calls newest first, MCP calls grouped last", async () => { + renderSessionDrawer(); + await waitFor(() => expect(sidebarEventNames()).toHaveLength(4)); + expect(sidebarEventNames()).toEqual(["llm-late", "llm-early", "tool-late", "tool-early"]); + }); + + it("switches to chronological order across LLM and MCP calls when Start time is selected", async () => { + renderSessionDrawer(); + await waitFor(() => expect(sidebarEventNames()).toHaveLength(4)); + + fireEvent.click(screen.getByText("Start time")); + + await waitFor(() => expect(sidebarEventNames()).toEqual(["llm-early", "tool-early", "llm-late", "tool-late"])); + + fireEvent.click(screen.getByText("Grouped")); + + await waitFor(() => expect(sidebarEventNames()).toEqual(["llm-late", "llm-early", "tool-late", "tool-early"])); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx index bf3360a5371..36299139a13 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx @@ -1,5 +1,5 @@ import { useEffect, useMemo, useState } from "react"; -import { Button, Drawer } from "antd"; +import { Button, Drawer, Segmented } from "antd"; import { CheckOutlined, CopyOutlined, LeftOutlined, RightOutlined } from "@ant-design/icons"; import { Bot, Sparkles, Wrench } from "lucide-react"; import { LogEntry } from "../columns"; @@ -11,7 +11,7 @@ import { LogDetailContent, GuardrailJumpLink } from "./LogDetailContent"; import { sessionSpendLogsCall } from "../../networking"; import { useQuery } from "@tanstack/react-query"; import { getSpendString } from "@/utils/dataUtils"; -import { normalizeGuardrailEntries } from "./utils"; +import { normalizeGuardrailEntries, sortSessionLogs, SessionLogSortMode } from "./utils"; import { DRAWER_WIDTH } from "./constants"; import { useLogDetails } from "@/app/(dashboard)/hooks/logDetails/useLogDetails"; @@ -117,6 +117,7 @@ export function LogDetailsDrawer({ }: LogDetailsDrawerProps) { const isSessionMode = Boolean(sessionId); const [selectedSessionRequestId, setSelectedSessionRequestId] = useState(null); + const [sessionSortMode, setSessionSortMode] = useState("grouped"); const [isSidebarCollapsed, setIsSidebarCollapsed] = useState(false); const [copiedLeftPanelId, setCopiedLeftPanelId] = useState(false); @@ -152,26 +153,20 @@ export function LogDetailsDrawer({ // backend omits total, so the truncation note reflects what was fetched. const total: number = firstPage.total ?? rows.length; - const logs = rows - .map((row) => ({ - ...row, - request_duration_ms: row.request_duration_ms ?? Date.parse(row.endTime) - Date.parse(row.startTime), - })) - .sort((a, b) => { - const aIsMcp = MCP_CALL_TYPES.includes(a.call_type) ? 1 : 0; - const bIsMcp = MCP_CALL_TYPES.includes(b.call_type) ? 1 : 0; - if (aIsMcp !== bIsMcp) return aIsMcp - bIsMcp; - // Newest first, matching the all-sessions logs overview. MCP calls - // stay grouped last (above), newest-first within that group too. - return new Date(b.startTime).getTime() - new Date(a.startTime).getTime(); - }); + const logs = rows.map((row) => ({ + ...row, + request_duration_ms: row.request_duration_ms ?? Date.parse(row.endTime) - Date.parse(row.startTime), + })); return { logs, total }; }, enabled: Boolean(open && isSessionMode && sessionId && accessToken), }); - const sessionLogs: LogEntry[] = sessionData?.logs ?? []; + const sessionLogs: LogEntry[] = useMemo( + () => sortSessionLogs(sessionData?.logs ?? [], sessionSortMode), + [sessionData, sessionSortMode], + ); // total reported by the backend; when the page cap truncates the fetch this // exceeds sessionLogs.length, which drives the "showing most recent" note. const sessionTotalCount = sessionData?.total ?? sessionLogs.length; @@ -391,6 +386,18 @@ export function LogDetailsDrawer({ Showing most recent {logsForList.length} of {sessionTotalCount} )} + {isSessionMode && ( + setSessionSortMode(value as SessionLogSortMode)} + /> + )}
diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts new file mode 100644 index 00000000000..cbe12f5c101 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { sortSessionLogs } from "./utils"; + +const llm = (id: string, startTime: string) => ({ request_id: id, call_type: "acompletion", startTime }); +const mcp = (id: string, startTime: string) => ({ request_id: id, call_type: "call_mcp_tool", startTime }); + +const ids = (rows: { request_id: string }[]) => rows.map((row) => row.request_id); + +describe("sortSessionLogs", () => { + const rows = [ + mcp("mcp-early", "2026-07-08T10:00:01.000Z"), + llm("llm-late", "2026-07-08T10:00:02.000Z"), + mcp("mcp-late", "2026-07-08T10:00:03.000Z"), + llm("llm-early", "2026-07-08T10:00:00.000Z"), + ]; + + it("grouped mode keeps MCP calls last, newest first within each group", () => { + expect(ids(sortSessionLogs(rows, "grouped"))).toEqual(["llm-late", "llm-early", "mcp-late", "mcp-early"]); + }); + + it("chronological mode interleaves all calls by start time, oldest first", () => { + expect(ids(sortSessionLogs(rows, "chronological"))).toEqual(["llm-early", "mcp-early", "llm-late", "mcp-late"]); + }); + + it("does not mutate the input array", () => { + const input = [...rows]; + sortSessionLogs(input, "chronological"); + expect(ids(input)).toEqual(ids(rows)); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts index 61301cf5b54..5a1a0e81f96 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts @@ -3,6 +3,27 @@ * These functions handle data formatting, validation, and guardrail calculations. */ +import { MCP_CALL_TYPES } from "../constants"; + +export type SessionLogSortMode = "grouped" | "chronological"; + +export function sortSessionLogs( + rows: T[], + mode: SessionLogSortMode, +): T[] { + if (mode === "chronological") { + return [...rows].sort((a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime()); + } + return [...rows].sort((a, b) => { + const aIsMcp = MCP_CALL_TYPES.includes(a.call_type) ? 1 : 0; + const bIsMcp = MCP_CALL_TYPES.includes(b.call_type) ? 1 : 0; + if (aIsMcp !== bIsMcp) return aIsMcp - bIsMcp; + // Newest first, matching the all-sessions logs overview. MCP calls + // stay grouped last (above), newest-first within that group too. + return new Date(b.startTime).getTime() - new Date(a.startTime).getTime(); + }); +} + /** * Formats data for display. If input is a string, attempts to parse as JSON. * @param input - Data to format (string or object) From df2d44bab1e2c2ffd3acf68bbc0abf6ab1160f85 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Wed, 8 Jul 2026 16:54:39 +1000 Subject: [PATCH 006/101] feat(ui): sort session sidebar by duration or start time --- .../LogDetailsDrawer.test.tsx | 12 +++---- .../LogDetailsDrawer/LogDetailsDrawer.tsx | 30 ++++++++-------- .../view_logs/LogDetailsDrawer/utils.test.ts | 36 +++++++++++++------ .../view_logs/LogDetailsDrawer/utils.ts | 23 +++++------- 4 files changed, 55 insertions(+), 46 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx index db0d168fcac..1d23fecb5da 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx @@ -53,13 +53,13 @@ const sessionLogs = [ model: "tool-early", call_type: "call_mcp_tool", startTime: "2026-07-08T10:00:01.000Z", - endTime: "2026-07-08T10:00:01.500Z", + endTime: "2026-07-08T10:00:06.000Z", }), makeLog({ request_id: "llm-late", model: "llm-late", startTime: "2026-07-08T10:00:02.000Z", - endTime: "2026-07-08T10:00:04.000Z", + endTime: "2026-07-08T10:00:05.000Z", }), makeLog({ request_id: "mcp-late", @@ -84,10 +84,10 @@ const sidebarEventNames = () => screen.queryAllByText(/^(llm-early|llm-late|tool-early|tool-late)$/).map((el) => el.textContent); describe("LogDetailsDrawer session sidebar sorting", () => { - it("defaults to grouped order: LLM calls newest first, MCP calls grouped last", async () => { + it("defaults to duration order, longest call first across LLM and MCP calls", async () => { renderSessionDrawer(); await waitFor(() => expect(sidebarEventNames()).toHaveLength(4)); - expect(sidebarEventNames()).toEqual(["llm-late", "llm-early", "tool-late", "tool-early"]); + expect(sidebarEventNames()).toEqual(["tool-early", "llm-late", "llm-early", "tool-late"]); }); it("switches to chronological order across LLM and MCP calls when Start time is selected", async () => { @@ -98,8 +98,8 @@ describe("LogDetailsDrawer session sidebar sorting", () => { await waitFor(() => expect(sidebarEventNames()).toEqual(["llm-early", "tool-early", "llm-late", "tool-late"])); - fireEvent.click(screen.getByText("Grouped")); + fireEvent.click(screen.getByText("Duration")); - await waitFor(() => expect(sidebarEventNames()).toEqual(["llm-late", "llm-early", "tool-late", "tool-early"])); + await waitFor(() => expect(sidebarEventNames()).toEqual(["tool-early", "llm-late", "llm-early", "tool-late"])); }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx index 36299139a13..79592216942 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx @@ -117,7 +117,7 @@ export function LogDetailsDrawer({ }: LogDetailsDrawerProps) { const isSessionMode = Boolean(sessionId); const [selectedSessionRequestId, setSelectedSessionRequestId] = useState(null); - const [sessionSortMode, setSessionSortMode] = useState("grouped"); + const [sessionSortMode, setSessionSortMode] = useState("duration"); const [isSidebarCollapsed, setIsSidebarCollapsed] = useState(false); const [copiedLeftPanelId, setCopiedLeftPanelId] = useState(false); @@ -173,9 +173,9 @@ export function LogDetailsDrawer({ const sessionTruncated = sessionTotalCount > sessionLogs.length; // Default selection for a freshly opened session: the most recent log (latest - // startTime). The list is sorted newest-first, but MCP calls are grouped last, - // so the latest log by time is not necessarily sessionLogs[0]; compute it - // explicitly. A clicked/remembered log still wins over this default. + // startTime). The list is ordered by the selected sort mode, so the latest + // log by time is not necessarily sessionLogs[0]; compute it explicitly. + // A clicked/remembered log still wins over this default. const mostRecentLog = useMemo( () => sessionLogs.reduce( @@ -387,16 +387,18 @@ export function LogDetailsDrawer({
)} {isSessionMode && ( - setSessionSortMode(value as SessionLogSortMode)} - /> +
+ Sort by + setSessionSortMode(value as SessionLogSortMode)} + /> +
)} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts index cbe12f5c101..f59a20529d0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.test.ts @@ -1,30 +1,44 @@ import { describe, expect, it } from "vitest"; import { sortSessionLogs } from "./utils"; -const llm = (id: string, startTime: string) => ({ request_id: id, call_type: "acompletion", startTime }); -const mcp = (id: string, startTime: string) => ({ request_id: id, call_type: "call_mcp_tool", startTime }); +const log = (id: string, startTime: string, endTime: string, request_duration_ms?: number) => ({ + request_id: id, + startTime, + endTime, + request_duration_ms, +}); const ids = (rows: { request_id: string }[]) => rows.map((row) => row.request_id); describe("sortSessionLogs", () => { const rows = [ - mcp("mcp-early", "2026-07-08T10:00:01.000Z"), - llm("llm-late", "2026-07-08T10:00:02.000Z"), - mcp("mcp-late", "2026-07-08T10:00:03.000Z"), - llm("llm-early", "2026-07-08T10:00:00.000Z"), + log("mid-duration", "2026-07-08T10:00:01.000Z", "2026-07-08T10:00:01.500Z", 2000), + log("longest", "2026-07-08T10:00:02.000Z", "2026-07-08T10:00:02.500Z", 5000), + log("shortest", "2026-07-08T10:00:03.000Z", "2026-07-08T10:00:03.500Z", 300), + log("earliest-no-duration-field", "2026-07-08T10:00:00.000Z", "2026-07-08T10:00:04.000Z"), ]; - it("grouped mode keeps MCP calls last, newest first within each group", () => { - expect(ids(sortSessionLogs(rows, "grouped"))).toEqual(["llm-late", "llm-early", "mcp-late", "mcp-early"]); + it("duration mode sorts longest call first, deriving duration from timestamps when the field is missing", () => { + expect(ids(sortSessionLogs(rows, "duration"))).toEqual([ + "longest", + "earliest-no-duration-field", + "mid-duration", + "shortest", + ]); }); - it("chronological mode interleaves all calls by start time, oldest first", () => { - expect(ids(sortSessionLogs(rows, "chronological"))).toEqual(["llm-early", "mcp-early", "llm-late", "mcp-late"]); + it("start_time mode sorts calls in the order they started", () => { + expect(ids(sortSessionLogs(rows, "start_time"))).toEqual([ + "earliest-no-duration-field", + "mid-duration", + "longest", + "shortest", + ]); }); it("does not mutate the input array", () => { const input = [...rows]; - sortSessionLogs(input, "chronological"); + sortSessionLogs(input, "duration"); expect(ids(input)).toEqual(ids(rows)); }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts index 5a1a0e81f96..d313d07361c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/utils.ts @@ -3,25 +3,18 @@ * These functions handle data formatting, validation, and guardrail calculations. */ -import { MCP_CALL_TYPES } from "../constants"; +export type SessionLogSortMode = "duration" | "start_time"; -export type SessionLogSortMode = "grouped" | "chronological"; +type SortableSessionLog = { startTime: string; endTime: string; request_duration_ms?: number }; -export function sortSessionLogs( - rows: T[], - mode: SessionLogSortMode, -): T[] { - if (mode === "chronological") { +const durationMs = (row: SortableSessionLog): number => + row.request_duration_ms ?? Date.parse(row.endTime) - Date.parse(row.startTime); + +export function sortSessionLogs(rows: T[], mode: SessionLogSortMode): T[] { + if (mode === "start_time") { return [...rows].sort((a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime()); } - return [...rows].sort((a, b) => { - const aIsMcp = MCP_CALL_TYPES.includes(a.call_type) ? 1 : 0; - const bIsMcp = MCP_CALL_TYPES.includes(b.call_type) ? 1 : 0; - if (aIsMcp !== bIsMcp) return aIsMcp - bIsMcp; - // Newest first, matching the all-sessions logs overview. MCP calls - // stay grouped last (above), newest-first within that group too. - return new Date(b.startTime).getTime() - new Date(a.startTime).getTime(); - }); + return [...rows].sort((a, b) => durationMs(b) - durationMs(a)); } /** From 5d89be551bbed49c493f2a1cea0bddf4ea8e468b Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Wed, 8 Jul 2026 17:06:38 +1000 Subject: [PATCH 007/101] fix(ui): fit session sort toggle inside sidebar column --- .../LogDetailsDrawer/LogDetailsDrawer.tsx | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx index 79592216942..92cf90fad63 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx @@ -387,18 +387,17 @@ export function LogDetailsDrawer({ )} {isSessionMode && ( -
- Sort by - setSessionSortMode(value as SessionLogSortMode)} - /> -
+ setSessionSortMode(value as SessionLogSortMode)} + /> )} From 6d2090a21b19d6277d44367983d9d70ee10e8a0c Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Wed, 8 Jul 2026 17:17:44 +1000 Subject: [PATCH 008/101] fix(ui): reset session sort mode when drawer closes --- .../LogDetailsDrawer.test.tsx | 21 ++++++++++++++++--- .../LogDetailsDrawer/LogDetailsDrawer.tsx | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx index 1d23fecb5da..5a49cccec70 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.test.tsx @@ -73,11 +73,13 @@ const sessionLogs = [ const renderSessionDrawer = () => { vi.mocked(sessionSpendLogsCall).mockResolvedValue({ data: sessionLogs, total: 4, total_pages: 1 }); const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - render( + const drawer = (open: boolean) => ( - {}} logEntry={null} sessionId="session-1" accessToken="token" /> - , + {}} logEntry={null} sessionId="session-1" accessToken="token" /> + ); + const { rerender } = render(drawer(true)); + return { rerender, drawer }; }; const sidebarEventNames = () => @@ -102,4 +104,17 @@ describe("LogDetailsDrawer session sidebar sorting", () => { await waitFor(() => expect(sidebarEventNames()).toEqual(["tool-early", "llm-late", "llm-early", "tool-late"])); }); + + it("resets the sort mode back to duration when the drawer is closed and reopened", async () => { + const { rerender, drawer } = renderSessionDrawer(); + await waitFor(() => expect(sidebarEventNames()).toHaveLength(4)); + + fireEvent.click(screen.getByText("Start time")); + await waitFor(() => expect(sidebarEventNames()).toEqual(["llm-early", "tool-early", "llm-late", "tool-late"])); + + rerender(drawer(false)); + rerender(drawer(true)); + + await waitFor(() => expect(sidebarEventNames()).toEqual(["tool-early", "llm-late", "llm-early", "tool-late"])); + }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx index 92cf90fad63..cc087a611b0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx @@ -217,6 +217,7 @@ export function LogDetailsDrawer({ setIsSidebarCollapsed(false); } else { if (isSessionMode) setSelectedSessionRequestId(null); + setSessionSortMode("duration"); setCopiedLeftPanelId(false); } }, [open, isSessionMode]); From bfff5e8d868312fcec9fe7fd9aaa3df14aa31ea3 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 8 Jul 2026 19:02:48 +0300 Subject: [PATCH 009/101] fix(mcp): log MCP tool calls returning isError=true as failures (#32238) An MCP tool call that completes with CallToolResult.isError=true correctly returns HTTP 200 per the MCP spec, but the shared post-call logging helper always fired async_success_handler, so the standard logging payload carried status=success and OTel (whose _parse_error only marks ERROR on status=failure) showed green spans for failed tools. The helper now checks the result after async_post_mcp_tool_call_hook runs (guardrails may flip isError there) and routes error results to the failure path: success gates are consumed so the @client wrapper cannot enqueue a success log, failure_handler and async_failure_handler fire with a new MCPToolResultError carrying the tool's first text content, and post_call_failure_hook records the failure the same way raised exceptions already do. Raised exceptions never reach the helper, so no double failure logging. HTTP wire behavior is unchanged Resolves LIT-4081 --- .../_experimental/mcp_server/exceptions.py | 15 + .../mcp_server/rest_endpoints.py | 36 ++- .../proxy/_experimental/mcp_server/server.py | 72 ++++- .../proxy/_experimental/mcp_server/utils.py | 19 ++ .../mcp_server/test_mcp_server.py | 304 ++++++++++++++++++ .../mcp_server/test_mcp_tool_search.py | 2 +- .../mcp_server/test_rest_endpoints.py | 6 +- 7 files changed, 440 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/exceptions.py b/litellm/proxy/_experimental/mcp_server/exceptions.py index b3f7ca9bbe2..3e3e549008d 100644 --- a/litellm/proxy/_experimental/mcp_server/exceptions.py +++ b/litellm/proxy/_experimental/mcp_server/exceptions.py @@ -73,3 +73,18 @@ class MCPUpstreamAuthError(Exception): detail=detail, headers={"www-authenticate": challenge} if challenge else None, ) + + +class MCPToolResultError(Exception): + """An MCP tool call completed with ``isError=True`` in its result. + + Never raised on the wire path: streamable HTTP MCP correctly returns tool + failures as HTTP 200 with ``result.isError: true`` per the MCP spec. This + exception only drives the standard failure logging (``status="failure"`` + payload, OTel ERROR span) for such results. + + Lives here rather than ``utils.py`` deliberately: tests reload ``utils`` + to re-read its env-derived constants, and a reload would fork this class + into two identities, breaking ``isinstance`` checks against instances + created before the reload. + """ diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index d482e537c5d..b917530dd52 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -8,6 +8,7 @@ from typing import ( Dict, List, Literal, + Mapping, Optional, Set, Tuple, @@ -78,7 +79,7 @@ if MCP_AVAILABLE: MCPInfo, MCPServer, _apply_toolset_scope, - _fire_mcp_success_logging, + _fire_mcp_tool_call_logging, _tool_name_matches, execute_mcp_tool, filter_tools_by_allowed_tools, @@ -86,23 +87,32 @@ if MCP_AVAILABLE: ######################################################## ############ MCP Server REST API Routes ################# - async def _safe_fire_mcp_success_logging( + async def _safe_fire_mcp_tool_call_logging( logging_obj: Optional[Any], result: Any, start_time: datetime, end_time: datetime, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + request_data: Optional[Mapping[str, object]] = None, ) -> None: if logging_obj is None: return logging_results = await asyncio.gather( - _fire_mcp_success_logging(logging_obj, result, start_time, end_time), + _fire_mcp_tool_call_logging( + logging_obj, + result, + start_time, + end_time, + user_api_key_auth=user_api_key_auth, + request_data=request_data, + ), return_exceptions=True, ) logging_error = logging_results[0] if isinstance(logging_error, asyncio.CancelledError): raise logging_error if isinstance(logging_error, BaseException): - verbose_logger.warning("MCP tool success logging failed (continuing): %s", logging_error) + verbose_logger.warning("MCP tool call logging failed (continuing): %s", logging_error) def _get_server_auth_header( server, @@ -872,7 +882,14 @@ if MCP_AVAILABLE: raw_headers=virtual_raw_headers, litellm_logging_obj=virtual_logging_obj, ) - await _safe_fire_mcp_success_logging(virtual_logging_obj, result, _tool_start_time, datetime.now()) + await _safe_fire_mcp_tool_call_logging( + virtual_logging_obj, + result, + _tool_start_time, + datetime.now(), + user_api_key_auth=user_api_key_dict, + request_data=data, + ) return result # Validate required parameters early @@ -955,7 +972,14 @@ if MCP_AVAILABLE: litellm_logging_obj=data.get("litellm_logging_obj"), requested_server_id=canonical_server_id, ) - await _safe_fire_mcp_success_logging(logging_obj, result, _tool_start_time, datetime.now()) + await _safe_fire_mcp_tool_call_logging( + logging_obj, + result, + _tool_start_time, + datetime.now(), + user_api_key_auth=user_api_key_dict, + request_data=data, + ) return result except MCPMissingUserEnvVarsError as e: verbose_logger.info( diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index fc847182a60..c03e49a1628 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -20,6 +20,7 @@ from typing import ( Callable, Dict, List, + Mapping, Optional, Set, Tuple, @@ -47,7 +48,10 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( get_request_base_url, ) -from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError +from litellm.proxy._experimental.mcp_server.exceptions import ( + MCPToolResultError, + MCPUpstreamAuthError, +) from litellm.proxy._experimental.mcp_server.mcp_context import ( _mcp_active_toolset_id, _mcp_gateway_initialize_instructions, @@ -60,6 +64,7 @@ from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_VERSION, MCPMissingUserEnvVarsError, add_server_prefix_to_name, + extract_mcp_tool_result_error_message, get_server_prefix, iter_known_server_prefixes, ) @@ -2743,12 +2748,40 @@ if MCP_AVAILABLE: return response - async def _fire_mcp_success_logging( + _MCP_CREDENTIAL_REQUEST_FIELDS = frozenset( + { + "raw_headers", + "mcp_auth_header", + "mcp_server_auth_headers", + "oauth2_headers", + "user_api_key_auth", + } + ) + + async def _fire_mcp_tool_call_logging( logging_obj: LiteLLMLoggingObj, result: Any, start_time: datetime, end_time: datetime, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + request_data: Optional[Mapping[str, object]] = None, ) -> None: + """Fire post-call logging for an executed MCP tool call. + + A result with ``isError=True`` is logged as a failure (``status="failure"`` + payload, so OTel marks the span ERROR) while the HTTP wire behavior stays + 200 + ``isError: true`` per the MCP spec. The error check runs after + ``async_post_mcp_tool_call_hook`` because guardrails may flip the result + to ``isError=True`` in that hook. Raised exceptions never reach here (the + ``@client`` wrapper and ``call_mcp_tool``'s except path log those), so + this cannot double-log a failure. + + ``request_data`` may carry credential-bearing fields (the REST path puts + ``raw_headers``, ``mcp_auth_header``, ``mcp_server_auth_headers``, and + ``oauth2_headers`` at the top level of its data dict), so those are + stripped before the dict is handed to ``post_call_failure_hook`` + callbacks. + """ logging_obj.post_call(original_response=result) await logging_obj.async_post_mcp_tool_call_hook( kwargs=logging_obj.model_call_details, @@ -2757,7 +2790,31 @@ if MCP_AVAILABLE: end_time=end_time, ) logging_obj.call_type = CallTypes.call_mcp_tool.value - await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time) + error_message = extract_mcp_tool_result_error_message(result) + if error_message is None: + await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time) + return + + logging_obj.has_run_logging(event_type="sync_success") + logging_obj.has_run_logging(event_type="async_success") + tool_error = MCPToolResultError(error_message) + logging_obj.failure_handler(tool_error, "", start_time, end_time) + await logging_obj.async_failure_handler(tool_error, "", start_time, end_time) + + if user_api_key_auth is None: + return + from litellm.proxy.proxy_server import proxy_logging_obj + + if proxy_logging_obj: + sanitized_request_data = { + key: value for key, value in (request_data or {}).items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS + } + await proxy_logging_obj.post_call_failure_hook( + request_data=sanitized_request_data, + original_exception=tool_error, + user_api_key_dict=user_api_key_auth, + route="/mcp/call_tool", + ) @client async def call_mcp_tool( @@ -2833,7 +2890,14 @@ if MCP_AVAILABLE: raise if litellm_logging_obj: - await _fire_mcp_success_logging(litellm_logging_obj, response, start_time, datetime.now()) + await _fire_mcp_tool_call_logging( + logging_obj=litellm_logging_obj, + result=response, + start_time=start_time, + end_time=datetime.now(), + user_api_key_auth=user_api_key_auth, + request_data=kwargs, + ) return response async def mcp_get_prompt( diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index c9c60030dbc..80a469b8c1a 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -415,6 +415,25 @@ def validate_mcp_server_name(server_name: str, raise_http_exception: bool = Fals raise Exception(error_message) +def extract_mcp_tool_result_error_message(result: object) -> Optional[str]: + """The first text content of an ``isError=True`` tool result, or ``None`` + when the result is not an error. + + Accepts both ``mcp.types.CallToolResult`` objects and their dict + equivalents, duck-typed so the ``mcp`` package is not required. + """ + is_error: object = result.get("isError") if isinstance(result, Mapping) else getattr(result, "isError", None) + if is_error is not True: + return None + content: object = result.get("content") if isinstance(result, Mapping) else getattr(result, "content", None) + if isinstance(content, (list, tuple)): + for item in content: + text: object = item.get("text") if isinstance(item, Mapping) else getattr(item, "text", None) + if isinstance(text, str) and text: + return text + return "MCP tool call returned isError=true" + + TOOL_DISPLAY_NAME_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index b24457deabd..83c19dfd7ca 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -8,8 +8,10 @@ from fastapi import HTTPException from mcp import ReadResourceResult, Resource from mcp.types import ( BlobResourceContents, + CallToolResult, Prompt, ResourceTemplate, + TextContent, TextResourceContents, ) @@ -6598,6 +6600,308 @@ async def test_get_active_submitted_mcp_server_ids_for_user_empty_user_id_skips_ prisma_client.db.litellm_mcpservertable.find_many.assert_not_awaited() +# --------------------------------------------------------------------------- # +# MCP tool-call isError failure logging +# --------------------------------------------------------------------------- # + + +def _call_tool_result(is_error: bool, text: str) -> CallToolResult: + return CallToolResult(content=[TextContent(type="text", text=text)], isError=is_error) + + +def _mock_mcp_logging_obj() -> MagicMock: + logging_obj = MagicMock() + logging_obj.model_call_details = {} + logging_obj.async_post_mcp_tool_call_hook = AsyncMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + return logging_obj + + +def test_extract_mcp_tool_result_error_message(): + from litellm.proxy._experimental.mcp_server.utils import ( + extract_mcp_tool_result_error_message, + ) + + assert extract_mcp_tool_result_error_message(_call_tool_result(True, "boom")) == "boom" + assert extract_mcp_tool_result_error_message(_call_tool_result(False, "ok")) is None + assert ( + extract_mcp_tool_result_error_message(CallToolResult(content=[], isError=True)) + == "MCP tool call returned isError=true" + ) + assert ( + extract_mcp_tool_result_error_message({"isError": True, "content": [{"type": "text", "text": "denied"}]}) + == "denied" + ) + assert extract_mcp_tool_result_error_message({"isError": False, "content": []}) is None + assert extract_mcp_tool_result_error_message({}) is None + + +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_iserror_logs_failure(): + """Regression test: a CallToolResult with isError=True must go + down the failure logging path (async_failure_handler + post_call_failure_hook), + never async_success_handler.""" + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + from litellm.proxy._experimental.mcp_server.exceptions import MCPToolResultError + + logging_obj = _mock_mcp_logging_obj() + proxy_logging_mock = MagicMock() + proxy_logging_mock.post_call_failure_hook = AsyncMock() + user_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user") + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_mock): + await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=_call_tool_result(True, "upstream exploded"), + start_time=datetime.now(), + end_time=datetime.now(), + user_api_key_auth=user_auth, + request_data={"litellm_call_id": "cid"}, + ) + + logging_obj.async_success_handler.assert_not_awaited() + logging_obj.failure_handler.assert_called_once() + logging_obj.async_failure_handler.assert_awaited_once() + tool_error = logging_obj.async_failure_handler.await_args.args[0] + assert isinstance(tool_error, MCPToolResultError) + assert str(tool_error) == "upstream exploded" + logging_obj.has_run_logging.assert_any_call(event_type="sync_success") + logging_obj.has_run_logging.assert_any_call(event_type="async_success") + proxy_logging_mock.post_call_failure_hook.assert_awaited_once() + hook_kwargs = proxy_logging_mock.post_call_failure_hook.await_args.kwargs + assert hook_kwargs["route"] == "/mcp/call_tool" + assert hook_kwargs["original_exception"] is tool_error + assert hook_kwargs["user_api_key_dict"] is user_auth + logging_obj.async_post_mcp_tool_call_hook.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_success_path_unchanged(): + """isError=False must keep today's behavior: success handler fires, no + failure logging, no post_call_failure_hook.""" + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + + logging_obj = _mock_mcp_logging_obj() + proxy_logging_mock = MagicMock() + proxy_logging_mock.post_call_failure_hook = AsyncMock() + result = _call_tool_result(False, "all good") + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_mock): + await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=result, + start_time=datetime.now(), + end_time=datetime.now(), + user_api_key_auth=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + request_data={}, + ) + + logging_obj.async_success_handler.assert_awaited_once() + assert logging_obj.async_success_handler.await_args.kwargs["result"] is result + logging_obj.async_failure_handler.assert_not_awaited() + logging_obj.failure_handler.assert_not_called() + proxy_logging_mock.post_call_failure_hook.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_iserror_without_auth_skips_failure_hook(): + """Without a UserAPIKeyAuth the failure handlers still fire but the proxy + post_call_failure_hook (which requires one) is skipped.""" + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + + logging_obj = _mock_mcp_logging_obj() + proxy_logging_mock = MagicMock() + proxy_logging_mock.post_call_failure_hook = AsyncMock() + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_mock): + await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result={"isError": True, "content": [{"type": "text", "text": "denied"}]}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + logging_obj.async_success_handler.assert_not_awaited() + logging_obj.async_failure_handler.assert_awaited_once() + assert str(logging_obj.async_failure_handler.await_args.args[0]) == "denied" + proxy_logging_mock.post_call_failure_hook.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_strips_credentials_from_failure_hook(): + """Credential-bearing request_data fields (raw request headers, upstream MCP + auth headers, OAuth tokens) must never reach post_call_failure_hook + callbacks; non-credential fields must survive untouched.""" + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + + logging_obj = _mock_mcp_logging_obj() + proxy_logging_mock = MagicMock() + proxy_logging_mock.post_call_failure_hook = AsyncMock() + user_auth = UserAPIKeyAuth(api_key="test-key", user_id="test-user") + request_data = { + "name": "explode", + "litellm_call_id": "cid", + "raw_headers": {"authorization": "Bearer sk-caller-secret"}, + "mcp_auth_header": "upstream-secret", + "mcp_server_auth_headers": {"srv": {"authorization": "Bearer srv-secret"}}, + "oauth2_headers": {"authorization": "Bearer oauth-secret"}, + "user_api_key_auth": user_auth, + } + + with patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_mock): + await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=_call_tool_result(True, "boom"), + start_time=datetime.now(), + end_time=datetime.now(), + user_api_key_auth=user_auth, + request_data=request_data, + ) + + proxy_logging_mock.post_call_failure_hook.assert_awaited_once() + hook_request_data = proxy_logging_mock.post_call_failure_hook.await_args.kwargs["request_data"] + assert hook_request_data == {"name": "explode", "litellm_call_id": "cid"} + assert "secret" not in str(hook_request_data) + + +def _real_mcp_logging_obj(call_id: str): + from litellm.litellm_core_utils.litellm_logging import Logging + + start_time = datetime.now() + logging_obj = Logging( + model="MCP: weather/get_forecast", + messages=[{"role": "user", "content": "tool call"}], + stream=False, + call_type="call_mcp_tool", + start_time=start_time, + litellm_call_id=call_id, + function_id="test-fn", + ) + logging_obj.update_environment_variables( + model="MCP: weather/get_forecast", + user="", + optional_params={}, + litellm_params={"api_base": ""}, + ) + logging_obj.model_call_details["mcp_tool_call_metadata"] = { + "name": "get_forecast", + "arguments": {"city": "Paris"}, + "mcp_server_name": "weather", + } + return logging_obj, start_time + + +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_iserror_builds_failure_payload(monkeypatch): + """The standard logging payload for an isError=True result must carry + status='failure' with the tool's error text, so OTel (whose _parse_error + keys off status) marks the MCP span ERROR.""" + import litellm + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + + logging_obj, start_time = _real_mcp_logging_obj("test-mcp-iserror-payload") + + await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=_call_tool_result(True, "upstream exploded"), + start_time=start_time, + end_time=datetime.now(), + ) + + payload = logging_obj.model_call_details["standard_logging_object"] + assert payload["status"] == "failure" + assert payload["error_str"] == "upstream exploded" + assert payload["error_information"]["error_class"] == "MCPToolResultError" + assert payload["metadata"]["mcp_tool_call_metadata"]["name"] == "get_forecast" + + +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_success_builds_success_payload(monkeypatch): + """isError=False still produces a status='success' payload.""" + import litellm + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + + logging_obj, start_time = _real_mcp_logging_obj("test-mcp-success-payload") + + await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=_call_tool_result(False, "all good"), + start_time=start_time, + end_time=datetime.now(), + ) + + payload = logging_obj.model_call_details["standard_logging_object"] + assert payload["status"] == "success" + + +@pytest.mark.asyncio +async def test_fire_mcp_tool_call_logging_iserror_emits_otel_error_span(monkeypatch): + """End-to-end regression for the OTel symptom: an isError=True tool + result must reach OTel as an MCP span with StatusCode.ERROR and the tool's + error message, while isError=False stays non-error.""" + pytest.importorskip("opentelemetry") + from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, + ) + from opentelemetry.trace.status import StatusCode + + import litellm + from litellm.integrations.otel import OpenTelemetryV2Config + from litellm.integrations.otel.logger import OpenTelemetryV2 + from litellm.integrations.otel.plumbing import providers + from litellm.proxy._experimental.mcp_server.server import ( + _fire_mcp_tool_call_logging, + ) + + cfg = OpenTelemetryV2Config(exporter="in_memory", legacy_compat=False) + exporter = InMemorySpanExporter() + tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter) + otel_logger = OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider) + + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", [otel_logger]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", [otel_logger]) + + logging_obj, start_time = _real_mcp_logging_obj("test-mcp-iserror-otel") + + await _fire_mcp_tool_call_logging( + logging_obj=logging_obj, + result=_call_tool_result(True, "upstream exploded"), + start_time=start_time, + end_time=datetime.now(), + ) + + (span,) = exporter.get_finished_spans() + assert span.name == "tools/call get_forecast" + assert span.status.status_code is StatusCode.ERROR + assert span.attributes["error.type"] == "MCPToolResultError" + assert "upstream exploded" in (span.status.description or "") + + @pytest.mark.asyncio async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow(): """ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py index 5c2a04456b0..b8f0b205831 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_tool_search.py @@ -433,7 +433,7 @@ class TestCallToolRestApiVirtualTools: return_value=fake_result, ) as mock_execute, patch( - "litellm.proxy._experimental.mcp_server.rest_endpoints._fire_mcp_success_logging", + "litellm.proxy._experimental.mcp_server.rest_endpoints._fire_mcp_tool_call_logging", new_callable=AsyncMock, side_effect=RuntimeError("logging failed"), ) as mock_fire_logging, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index e114f46e866..3d9afd8f250 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -1559,7 +1559,7 @@ class TestCallToolRestAPI: fire_logging = AsyncMock(side_effect=RuntimeError("logging failed")) monkeypatch.setattr( rest_endpoints, - "_fire_mcp_success_logging", + "_fire_mcp_tool_call_logging", fire_logging, raising=False, ) @@ -1590,13 +1590,13 @@ class TestCallToolRestAPI: fire_logging = AsyncMock(side_effect=asyncio.CancelledError()) monkeypatch.setattr( rest_endpoints, - "_fire_mcp_success_logging", + "_fire_mcp_tool_call_logging", fire_logging, raising=False, ) with pytest.raises(asyncio.CancelledError): - await rest_endpoints._safe_fire_mcp_success_logging( + await rest_endpoints._safe_fire_mcp_tool_call_logging( object(), {"result": "ok"}, datetime.now(), datetime.now() ) From c2d8a17692cb4dbaacccf9ecb3d678a8e4788db8 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:01:41 -0700 Subject: [PATCH 010/101] test(responses): replace perma-skip azure shell e2e with offline coverage (#32444) --- .../base_responses_api.py | 9 +- .../test_azure_responses_api.py | 5 - .../azure_shell_tool.json | 14 ++ .../test_responses_api_request_body.py | 150 ++++++++++++++---- 4 files changed, 142 insertions(+), 36 deletions(-) create mode 100644 tests/test_litellm/expected_responses_api_request/azure_shell_tool.json diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 7d2e30f8372..407091a65b3 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -746,7 +746,8 @@ class BaseResponsesAPITest(ABC): E2E test for Shell tool on OpenAI Responses API. Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}]; validates that the request is accepted and returns a valid response. - Only runs for OpenAI/Azure (Responses API with shell support). + Only runs for OpenAI; offline coverage for the Azure route lives in + tests/test_litellm/responses/test_responses_api_request_body.py. """ base_completion_call_args = self.get_base_completion_call_args() model = ( @@ -754,8 +755,10 @@ class BaseResponsesAPITest(ABC): or base_completion_call_args.get("model") or "" ) - if "openai/" not in str(model) and "azure/" not in str(model): - pytest.skip("Shell tool e2e is only run for OpenAI/Azure Responses API") + if "openai/" not in str(model): + pytest.skip( + "Shell tool e2e is OpenAI-only; no Azure deployment supports the shell tool yet, re-enable once one exists" + ) tools = [{"type": "shell", "environment": {"type": "container_auto"}}] input_msg = "List files in /mnt/data and show python --version." try: diff --git a/tests/llm_responses_api_testing/test_azure_responses_api.py b/tests/llm_responses_api_testing/test_azure_responses_api.py index fed9e9e11f0..ccef8cbf1e7 100644 --- a/tests/llm_responses_api_testing/test_azure_responses_api.py +++ b/tests/llm_responses_api_testing/test_azure_responses_api.py @@ -2,7 +2,6 @@ import os import sys import pytest import asyncio -from typing import Optional from unittest.mock import patch, AsyncMock sys.path.insert(0, os.path.abspath("../..")) @@ -30,10 +29,6 @@ class TestAzureResponsesAPITest(BaseResponsesAPITest): "api_version": "2025-03-01-preview", } - def get_advanced_model_for_shell_tool(self) -> Optional[str]: - """If specified, overrides the model used by test_responses_api_shell_tool_streaming_sees_shell_output (e.g. openai/gpt-5.2 for shell support).""" - return "azure/gpt-5-mini" - @pytest.mark.asyncio async def test_azure_responses_api_preview_api_version(): diff --git a/tests/test_litellm/expected_responses_api_request/azure_shell_tool.json b/tests/test_litellm/expected_responses_api_request/azure_shell_tool.json new file mode 100644 index 00000000000..b716c518106 --- /dev/null +++ b/tests/test_litellm/expected_responses_api_request/azure_shell_tool.json @@ -0,0 +1,14 @@ +{ + "model": "gpt-5-mini", + "input": "List files in /mnt/data and run python --version.", + "tools": [ + { + "type": "shell", + "environment": { + "type": "container_auto" + } + } + ], + "tool_choice": "auto", + "max_output_tokens": 256 +} diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py index e312a11e893..c39ba75bd97 100644 --- a/tests/test_litellm/responses/test_responses_api_request_body.py +++ b/tests/test_litellm/responses/test_responses_api_request_body.py @@ -1,6 +1,7 @@ """ Test that litellm.responses() / litellm.aresponses() send the expected request body -over the wire. Expected JSON bodies are stored in expected_responses_api_request/. +over the wire and surface provider errors correctly. Expected JSON bodies are stored +in expected_responses_api_request/. """ import json @@ -18,24 +19,20 @@ def _expected_dir() -> Path: return Path(__file__).resolve().parent.parent / "expected_responses_api_request" -@pytest.mark.asyncio -async def test_aresponses_context_management_and_shell_request_body_matches_expected(): - """ - Call litellm.aresponses() with context_management and shell tool; - assert the httpx POST request body matches the expected JSON. - """ - expected_path = _expected_dir() / "context_management_and_shell.json" +def _load_expected_body(filename: str) -> dict: + expected_path = _expected_dir() / filename assert expected_path.exists(), f"Expected file not found: {expected_path}" with open(expected_path) as f: - expected_body = json.load(f) + return json.load(f) - # Minimal Responses API response so parsing succeeds - mock_response = { - "id": "resp_ctx_shell_test", + +def _minimal_responses_api_payload(response_id: str, model: str) -> dict: + return { + "id": response_id, "object": "response", "created_at": 1734366691, "status": "completed", - "model": "gpt-4o", + "model": model, "output": [ { "type": "message", @@ -69,21 +66,41 @@ async def test_aresponses_context_management_and_shell_request_body_matches_expe "user": None, } - class MockResponse: - def __init__(self, json_data, status_code=200): - self._json_data = json_data - self.status_code = status_code - self.text = json.dumps(json_data) - self.headers = httpx.Headers({}) - def json(self): - return self._json_data +class MockResponse: + def __init__(self, json_data, status_code=200): + self._json_data = json_data + self.status_code = status_code + self.text = json.dumps(json_data) + self.headers = httpx.Headers({}) + + def json(self): + return self._json_data + + +def _assert_request_body_matches(request_body: dict, expected_body: dict) -> None: + for key, expected_value in expected_body.items(): + assert key in request_body, f"Missing key in request body: {key}" + assert ( + request_body[key] == expected_value + ), f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}" + + +@pytest.mark.asyncio +async def test_aresponses_context_management_and_shell_request_body_matches_expected(): + """ + Call litellm.aresponses() with context_management and shell tool; + assert the httpx POST request body matches the expected JSON. + """ + expected_body = _load_expected_body("context_management_and_shell.json") with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock, ) as mock_post: - mock_post.return_value = MockResponse(mock_response, 200) + mock_post.return_value = MockResponse( + _minimal_responses_api_payload("resp_ctx_shell_test", "gpt-4o"), 200 + ) await litellm.aresponses( model="openai/gpt-4o", @@ -95,10 +112,87 @@ async def test_aresponses_context_management_and_shell_request_body_matches_expe ) mock_post.assert_called_once() - request_body = mock_post.call_args.kwargs["json"] + _assert_request_body_matches(mock_post.call_args.kwargs["json"], expected_body) - for key, expected_value in expected_body.items(): - assert key in request_body, f"Missing key in request body: {key}" - assert ( - request_body[key] == expected_value - ), f"Mismatch for key {key}: got {request_body[key]!r}, expected {expected_value!r}" + +@pytest.mark.asyncio +async def test_aresponses_azure_shell_tool_request_body_matches_expected(): + """ + Call litellm.aresponses() on the Azure route with the shell tool; + assert the httpx POST request body carries the shell tool verbatim. + """ + expected_body = _load_expected_body("azure_shell_tool.json") + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.return_value = MockResponse( + _minimal_responses_api_payload("resp_azure_shell_test", "gpt-5-mini"), 200 + ) + + await litellm.aresponses( + model="azure/gpt-5-mini", + api_base="https://fake-resource.openai.azure.com", + api_key="fake-api-key", + api_version="2025-03-01-preview", + input=expected_body["input"], + tools=expected_body["tools"], + tool_choice=expected_body["tool_choice"], + max_output_tokens=expected_body["max_output_tokens"], + ) + + mock_post.assert_called_once() + _assert_request_body_matches(mock_post.call_args.kwargs["json"], expected_body) + + +@pytest.mark.asyncio +async def test_aresponses_azure_shell_tool_400_maps_to_bad_request_error(): + """ + Azure rejects the shell tool for unsupported deployments with a 400; + litellm must surface that as litellm.BadRequestError carrying the provider message. + """ + error_body = { + "error": { + "message": "Tool of type 'shell' is not supported with this model.", + "type": "invalid_request_error", + "param": "tools", + "code": None, + } + } + + def _raise_azure_400(*args, **kwargs): + response = httpx.Response( + status_code=400, + json=error_body, + request=httpx.Request( + "POST", + kwargs.get( + "url", + "https://fake-resource.openai.azure.com/openai/responses", + ), + ), + ) + response.raise_for_status() + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + ) as mock_post: + mock_post.side_effect = _raise_azure_400 + + with pytest.raises(litellm.BadRequestError) as excinfo: + await litellm.aresponses( + model="azure/gpt-5-mini", + api_base="https://fake-resource.openai.azure.com", + api_key="fake-api-key", + api_version="2025-03-01-preview", + input="List files in /mnt/data and run python --version.", + tools=[{"type": "shell", "environment": {"type": "container_auto"}}], + tool_choice="auto", + max_output_tokens=256, + ) + + assert excinfo.value.status_code == 400 + assert "shell" in str(excinfo.value).lower() + assert "not supported" in str(excinfo.value).lower() From f982b67d78c65d335144d54b8c7c831fcab903f3 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 8 Jul 2026 10:36:00 -0700 Subject: [PATCH 011/101] fix(proxy): harden secret name validation for external secret manager integrations (LIT-4201) (#32092) key_alias can become the secret name used by external secret manager integrations (HashiCorp Vault, CyberArk Conjur) when store_virtual_keys is enabled. Add raise_if_unsafe_secret_name, a shared validation check applied unconditionally before a secret name reaches either integration or the /key/generate, /key/update, and /key/regenerate API boundary, independent of the existing enable_key_alias_format_validation opt-in flag. Also hardens the Vault URL builder to percent-encode reserved characters in secret_name (preserving "/" and "@"), and switches the Conjur policy body to a real YAML serializer instead of raw string interpolation. --- .../key_management_endpoints.py | 24 +++++- .../secret_managers/base_secret_manager.py | 15 ++++ .../cyberark_secret_manager.py | 8 +- .../hashicorp_secret_manager.py | 3 +- tests/litellm_utils_tests/test_cyberark.py | 77 +++++++++++++++++++ tests/litellm_utils_tests/test_hashicorp.py | 27 +++++++ .../test_key_management_endpoints.py | 29 ++++++- .../test_base_secret_manager.py | 59 ++++++++++++++ 8 files changed, 234 insertions(+), 8 deletions(-) create mode 100644 tests/test_litellm/secret_managers/test_base_secret_manager.py diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 63f4b731871..bf64f537c7f 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -111,6 +111,7 @@ from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) from litellm.router import Router +from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_name from litellm.secret_managers.main import get_secret from litellm.types.proxy.management_endpoints.key_management_endpoints import ( BulkUpdateKeyRequest, @@ -6291,8 +6292,13 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None: """ Validate the format of the key_alias. - Gated behind ``litellm.enable_key_alias_format_validation`` (default **False**). - When disabled, no validation is performed so existing workflows are not broken. + A baseline validation always runs, regardless of + ``litellm.enable_key_alias_format_validation``. + + The remaining charset/length rules are gated behind + ``litellm.enable_key_alias_format_validation`` (default **False**). When disabled, + only the baseline validation above is performed, so existing workflows are not + broken. Rules (when enabled): - None is OK (no alias). @@ -6300,10 +6306,20 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None: - start/end with alphanumeric - only allow a-zA-Z0-9_-/.@ """ - if not litellm.enable_key_alias_format_validation: + if key_alias is None: return - if key_alias is None: + try: + raise_if_unsafe_secret_name(key_alias) + except ValueError: + raise ProxyException( + message="Invalid key_alias", + type=ProxyErrorTypes.bad_request_error, + param="key_alias", + code=400, + ) + + if not litellm.enable_key_alias_format_validation: return if not _KEY_ALIAS_PATTERN.match(key_alias): diff --git a/litellm/secret_managers/base_secret_manager.py b/litellm/secret_managers/base_secret_manager.py index d33d76093c9..2bb8dc73138 100644 --- a/litellm/secret_managers/base_secret_manager.py +++ b/litellm/secret_managers/base_secret_manager.py @@ -1,3 +1,4 @@ +import re from abc import ABC, abstractmethod from typing import Any, Dict, Optional, Union @@ -5,6 +6,20 @@ import httpx from litellm import verbose_logger +_UNSAFE_SECRET_NAME_PATTERN = re.compile(r"(^|/)\.\.(/|$)|[\x00-\x1f\x7f-\x9f…

]") + + +def raise_if_unsafe_secret_name(secret_name: str) -> None: + """ + Validate a secret name before it is used by a secret manager integration. + + Rejects ".." only as a path segment (bounded by "/" or the start/end of the + string, e.g. "../x", "x/..", or exactly ".."), not as a plain substring, so + names like "release-1.0..2" are not rejected. + """ + if _UNSAFE_SECRET_NAME_PATTERN.search(secret_name): + raise ValueError(f"Invalid secret_name {secret_name!r}") + class BaseSecretManager(ABC): """ diff --git a/litellm/secret_managers/cyberark_secret_manager.py b/litellm/secret_managers/cyberark_secret_manager.py index faf6224757f..2b888cb85f6 100644 --- a/litellm/secret_managers/cyberark_secret_manager.py +++ b/litellm/secret_managers/cyberark_secret_manager.py @@ -4,6 +4,7 @@ from typing import Any, Dict, Optional, Union from urllib.parse import quote import httpx +import yaml import litellm from litellm._logging import verbose_logger @@ -15,7 +16,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import KeyManagementSystem -from .base_secret_manager import BaseSecretManager +from .base_secret_manager import BaseSecretManager, raise_if_unsafe_secret_name from .main import str_to_bool @@ -125,8 +126,11 @@ class CyberArkSecretManager(BaseSecretManager): """ # In production, we'd check if the variable exists first # For now, we'll attempt to create it and ignore if it already exists + raise_if_unsafe_secret_name(secret_name) policy_url = f"{self.conjur_addr}/policies/{self.conjur_account}/policy/root" - policy_yaml = f"- !variable {secret_name}\n" + # Use a real YAML serializer to build the scalar safely. + quoted_name = yaml.safe_dump(secret_name, default_style='"').strip() + policy_yaml = f"- !variable {quoted_name}\n" try: client = _get_httpx_client(params={"ssl_verify": self.ssl_verify}) diff --git a/litellm/secret_managers/hashicorp_secret_manager.py b/litellm/secret_managers/hashicorp_secret_manager.py index bd1b1097347..039aecb9e58 100644 --- a/litellm/secret_managers/hashicorp_secret_manager.py +++ b/litellm/secret_managers/hashicorp_secret_manager.py @@ -14,7 +14,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.proxy._types import KeyManagementSystem -from .base_secret_manager import BaseSecretManager +from .base_secret_manager import BaseSecretManager, raise_if_unsafe_secret_name class HashicorpSecretManager(BaseSecretManager): @@ -220,6 +220,7 @@ class HashicorpSecretManager(BaseSecretManager): - With custom mount: http://127.0.0.1:8200/v1/kv/data/mykey - With path prefix: http://127.0.0.1:8200/v1/secret/data/myapp/mykey """ + raise_if_unsafe_secret_name(secret_name) resolved_namespace = self._sanitize_path_component(namespace if namespace is not None else self.vault_namespace) resolved_mount = self._sanitize_path_component(mount_name if mount_name is not None else self.vault_mount_name) if resolved_mount is None: diff --git a/tests/litellm_utils_tests/test_cyberark.py b/tests/litellm_utils_tests/test_cyberark.py index 67575d3e781..71daf35a265 100644 --- a/tests/litellm_utils_tests/test_cyberark.py +++ b/tests/litellm_utils_tests/test_cyberark.py @@ -5,6 +5,7 @@ Integration test for CyberArk Conjur Secret Manager. import os import sys import pytest +import yaml from dotenv import load_dotenv load_dotenv() @@ -42,6 +43,82 @@ def create_mock_response(status_code: int, text: str = ""): return mock_response +@pytest.mark.asyncio +async def test_cyberark_write_secret_rejects_yaml_injection(): + """ + Regression test: async_write_secret must reject a secret_name that is not + safe to embed in the Conjur policy body, before any HTTP call is made. + """ + with patch("litellm.proxy.proxy_server.premium_user", True): + malicious_secret_name = "foo\n- !grant\n role: !!admin\n member: attacker" + + mock_sync_client = MagicMock() + mock_async_client = AsyncMock() + + with ( + patch( + "litellm.secret_managers.cyberark_secret_manager._get_httpx_client", + return_value=mock_sync_client, + ), + patch( + "litellm.secret_managers.cyberark_secret_manager.get_async_httpx_client", + return_value=mock_async_client, + ), + ): + cyberark_manager = CyberArkSecretManager() + + response = await cyberark_manager.async_write_secret( + secret_name=malicious_secret_name, + secret_value="sk-1234", + ) + + assert response["status"] == "error" + assert "Invalid secret_name" in response["message"] + # The malicious policy YAML must never reach the wire. + mock_sync_client.client.post.assert_not_called() + mock_async_client.post.assert_not_called() + + +@pytest.mark.parametrize( + "secret_name", + [ + "foo: bar", + "foo # bar", + "plain-alias", + "team/user@example.com", + ], +) +def test_cyberark_ensure_variable_exists_escapes_yaml_metacharacters(secret_name): + """ + Regression test: _ensure_variable_exists must escape secret_name (not just + denylist-check it) so the policy body always parses back to exactly one + '!variable' scalar node holding the untouched secret_name. + """ + with patch("litellm.proxy.proxy_server.premium_user", True): + captured = {} + + def _capture_post(url, headers=None, content=None): + captured["content"] = content + return create_mock_response(status_code=201, text="") + + mock_sync_client = MagicMock() + mock_sync_client.client.post.side_effect = _capture_post + + with patch( + "litellm.secret_managers.cyberark_secret_manager._get_httpx_client", + return_value=mock_sync_client, + ): + cyberark_manager = CyberArkSecretManager() + cyberark_manager._ensure_variable_exists(secret_name) + + policy_yaml = captured["content"] + parsed = yaml.compose(policy_yaml) + assert len(parsed.value) == 1 + node = parsed.value[0] + assert node.tag == "!variable" + assert node.value == secret_name + + @pytest.mark.asyncio async def test_cyberark_write_and_read_secret(): """ diff --git a/tests/litellm_utils_tests/test_hashicorp.py b/tests/litellm_utils_tests/test_hashicorp.py index 3bdf11ea565..9aff7ddc10e 100644 --- a/tests/litellm_utils_tests/test_hashicorp.py +++ b/tests/litellm_utils_tests/test_hashicorp.py @@ -409,6 +409,33 @@ def test_hashicorp_custom_mount_and_prefix(hashicorp_secret_manager): hashicorp_secret_manager.vault_namespace = original_namespace +@pytest.mark.parametrize( + "malicious_secret_name", + [ + "../../../other-app/creds", + "litellm/../../secret", + "foo\nbar", + "foo
bar", + "foo
bar", + "foo\x85bar", + ], +) +def test_hashicorp_get_url_rejects_path_traversal(monkeypatch, malicious_secret_name): + """ + Regression test: get_url must reject an invalid secret_name instead of + building a URL from it. + + Uses monkeypatch + a directly-constructed manager (not the shared + hashicorp_secret_manager fixture) so this runs in CI without real Vault + credentials configured; get_url performs no I/O. + """ + monkeypatch.setenv("HCP_VAULT_TOKEN", "test-token-for-get-url-only") + manager = HashicorpSecretManager() + + with pytest.raises(ValueError): + manager.get_url(malicious_secret_name) + + mock_old_vault_response = { "request_id": "80fafb6a-e96a-4c5b-29fa-ff505ac72201", "lease_id": "", diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index d707421aeb6..2fe7725fd12 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -9023,7 +9023,7 @@ class TestValidateKeyAliasFormat: litellm.enable_key_alias_format_validation = False def test_validation_skipped_when_flag_disabled(self): - """When enable_key_alias_format_validation is False (default), no validation occurs.""" + """When enable_key_alias_format_validation is False (default), no charset/length validation occurs.""" from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, ) @@ -9034,6 +9034,33 @@ class TestValidateKeyAliasFormat: _validate_key_alias_format("!invalid!") _validate_key_alias_format("a" * 256) + @pytest.mark.parametrize( + "unsafe_alias", + [ + "../../../other-app/creds", + "litellm/../../secret", + "foo\n- !grant\n role: !!admin\n member: attacker", + "foo\rbar", + "foo\x00bar", + ], + ) + def test_validate_key_alias_format_rejects_traversal_and_control_chars_even_when_flag_disabled( + self, unsafe_alias + ): + """ + Regression test: this check must reject an invalid key_alias unconditionally, + even when enable_key_alias_format_validation (the separate, opt-in charset + rule) is disabled. + """ + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _validate_key_alias_format, + ) + + with pytest.raises(ProxyException) as exc: + _validate_key_alias_format(unsafe_alias) + assert str(exc.value.code) == "400" + assert "Invalid key_alias" in str(exc.value.message) + def test_validate_key_alias_format_valid(self): from litellm.proxy.management_endpoints.key_management_endpoints import ( _validate_key_alias_format, diff --git a/tests/test_litellm/secret_managers/test_base_secret_manager.py b/tests/test_litellm/secret_managers/test_base_secret_manager.py new file mode 100644 index 00000000000..cba6a99ab7f --- /dev/null +++ b/tests/test_litellm/secret_managers/test_base_secret_manager.py @@ -0,0 +1,59 @@ +""" +Test raise_if_unsafe_secret_name, the shared guard applied before secret_name +reaches a secret manager backend. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) # Adds the parent directory to the system path + +from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_name + + +@pytest.mark.parametrize( + "secret_name", + [ + "..", + "../../../other-app/creds", + "litellm/../../secret", + "foo/../bar", + "foo/..", + "../foo", + "foo\nbar", + "foo\rbar", + "foo\x00bar", + "foo\x7fbar", + "foo\x85bar", + "foo
bar", + "foo
bar", + ], +) +def test_raise_if_unsafe_secret_name_rejects_traversal_and_line_breaks(secret_name): + with pytest.raises(ValueError): + raise_if_unsafe_secret_name(secret_name) + + +@pytest.mark.parametrize( + "secret_name", + [ + "plain-alias", + "my-key-123", + "prod/my-service-key", + "team/user@example.com", + "foo: bar", + "foo # bar", + "foo?evil=1", + "foo#bar", + "a" * 500, + "release-1.0..2", + "my..key", + "..foo", + "foo..", + "v2.0..1-beta", + ], +) +def test_raise_if_unsafe_secret_name_allows_legitimate_aliases(secret_name): + raise_if_unsafe_secret_name(secret_name) From c0327cded4f3631e09adf63ad8b488f1333a7ac1 Mon Sep 17 00:00:00 2001 From: David Katz Date: Wed, 8 Jul 2026 09:39:48 -0400 Subject: [PATCH 012/101] fix(mcp): pair token-endpoint client_secret with the same source as client_id On re-auth against a server with a persisted DCR client, register_client_with_server short-circuits and returns a placeholder client_secret ("dummy") that the browser echoes back to /token. exchange_token_with_server overrode the caller's client_id with the persisted one but still fell back to the caller's secret when the server had none stored, so a persisted public PKCE client (which has no secret) was paired with the literal string "dummy" and the IdP rejected the exchange with 401 on every re-authorization; the proxy surfaced that as a 500. First connects and brand-new servers worked because a real DCR registration ran and no placeholder existed. Resolve the secret from the server whenever the server's client_id wins, so a secretless public client sends no client_secret at all --- .../mcp_server/discoverable_endpoints.py | 6 +- .../mcp_server/test_discoverable_endpoints.py | 59 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index fa1f73cea77..d4dbee37cdc 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -581,8 +581,12 @@ async def exchange_token_with_server( if mcp_server.token_url is None: raise HTTPException(status_code=400, detail="MCP server token url is not set") + # The id and secret must come from the same source. When the server-side client_id wins, + # falling back to the caller's secret pairs the persisted client with a foreign secret; the + # register short-circuit hands clients a placeholder secret ("dummy"), so a re-auth against a + # persisted public PKCE client (no stored secret) would send that placeholder and the IdP 401s. resolved_client_id = mcp_server.client_id if mcp_server.client_id else client_id - resolved_client_secret = mcp_server.client_secret if mcp_server.client_secret else client_secret + resolved_client_secret = mcp_server.client_secret if mcp_server.client_id else client_secret try: client_auth = build_token_endpoint_client_auth( auth_method=mcp_server.token_endpoint_auth_method, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index c808b17678a..19d030f17c4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -4156,3 +4156,62 @@ async def test_store_per_user_token_server_side_skips_invalidate_when_db_write_f invalidate_mock.assert_not_awaited() cache_set_mock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_token_exchange_pairs_client_secret_with_server_client_id(): + """Re-auth regression: the register short-circuit hands the browser a placeholder + ``client_secret: "dummy"``, which the browser echoes back to /token. The server-side + persisted client_id wins the resolution, so the secret must come from the same (server) + source; pairing the persisted public PKCE client (no stored secret) with the caller's + placeholder makes the IdP reject the exchange with 401 on every re-auth.""" + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="srv-1", + name="srv-1", + server_name="srv-1", + alias="srv-1", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="persisted-client", + client_secret=None, + authorization_url="https://provider.example/oauth/authorize", + token_url="https://provider.example/oauth/token", + ) + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + mock_response = MagicMock() + mock_response.raise_for_status = MagicMock() + mock_response.json.return_value = {"access_token": "at", "token_type": "Bearer"} + mock_async_client = MagicMock() + mock_async_client.post = AsyncMock(return_value=mock_response) + + with patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=mock_async_client, + ): + await exchange_token_with_server( + request=mock_request, + mcp_server=server, + grant_type="authorization_code", + code="auth-code", + redirect_uri="https://litellm.example.com/ui/mcp/oauth/callback", + client_id="srv-1", + client_secret="dummy", + code_verifier="verifier", + ) + + sent = mock_async_client.post.call_args.kwargs["data"] + assert sent["client_id"] == "persisted-client" + assert "client_secret" not in sent From 33aaea363c13e8cc576f1732673308349945e7f9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 8 Jul 2026 10:45:20 -0700 Subject: [PATCH 013/101] ci: add OSS daily branch workflow --- .github/workflows/create_daily_oss_branch.yml | 61 ++++++++++ .github/workflows/oss_daily_guardrails.yml | 109 ++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 .github/workflows/create_daily_oss_branch.yml create mode 100644 .github/workflows/oss_daily_guardrails.yml diff --git a/.github/workflows/create_daily_oss_branch.yml b/.github/workflows/create_daily_oss_branch.yml new file mode 100644 index 00000000000..43de4a0e75f --- /dev/null +++ b/.github/workflows/create_daily_oss_branch.yml @@ -0,0 +1,61 @@ +name: Create Daily OSS Branch + +on: + schedule: + - cron: "0 16 * * 1-5" # 9am PT during daylight saving time, weekdays. + workflow_dispatch: + inputs: + date: + description: "Branch date in YYYY_MM_DD format. Defaults to today's UTC date." + required: false + type: string + +permissions: + contents: write + +jobs: + create-oss-branch: + if: github.repository == 'BerriAI/litellm' + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Create dated OSS branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REQUESTED_DATE: ${{ inputs.date }} + run: | + set -euo pipefail + + if [ -n "${REQUESTED_DATE}" ]; then + if ! echo "${REQUESTED_DATE}" | grep -Eq '^[0-9]{4}_[0-9]{2}_[0-9]{2}$'; then + echo "::error::date must use YYYY_MM_DD format, got '${REQUESTED_DATE}'" + exit 1 + fi + BRANCH_DATE="${REQUESTED_DATE}" + else + BRANCH_DATE="$(date -u +'%Y_%m_%d')" + fi + + BRANCH_NAME="litellm_oss_daily_${BRANCH_DATE}" + echo "Creating branch: ${BRANCH_NAME}" + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + git fetch origin main "${BRANCH_NAME}" || true + + if git show-ref --verify --quiet "refs/remotes/origin/${BRANCH_NAME}"; then + echo "Branch ${BRANCH_NAME} already exists. Skipping creation." + exit 0 + fi + + git checkout -b "${BRANCH_NAME}" origin/main + git push "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "${BRANCH_NAME}" + echo "Successfully created and pushed branch: ${BRANCH_NAME}" diff --git a/.github/workflows/oss_daily_guardrails.yml b/.github/workflows/oss_daily_guardrails.yml new file mode 100644 index 00000000000..173b7cd4e41 --- /dev/null +++ b/.github/workflows/oss_daily_guardrails.yml @@ -0,0 +1,109 @@ +name: OSS Daily Guardrails + +on: + push: + branches: + - "litellm_oss_daily_20*" + pull_request: + branches: + - "litellm_oss_daily_20*" + - litellm_internal_staging + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + sensitive-file-guard: + name: Block sensitive OSS daily changes + if: startsWith(github.ref_name, 'litellm_oss_daily_20') || startsWith(github.head_ref, 'litellm_oss_daily_20') || startsWith(github.base_ref, 'litellm_oss_daily_20') + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Check for sensitive file changes + env: + EVENT_NAME: ${{ github.event_name }} + BASE_REF_NAME: ${{ github.base_ref }} + HEAD_REF_NAME: ${{ github.head_ref }} + run: | + set -euo pipefail + + if [ "${EVENT_NAME}" = "pull_request" ] && echo "${HEAD_REF_NAME}" | grep -Eq '^litellm_oss_daily_20[0-9]{2}_[0-9]{2}_[0-9]{2}$'; then + # Final daily OSS branch PR into staging: review only the OSS delta + # accumulated on top of main, not unrelated main/staging drift. + BASE_REF="origin/main" + git fetch origin main + elif [ "${EVENT_NAME}" = "pull_request" ]; then + # PR targeting the daily OSS branch: review the incoming PR delta. + BASE_REF="origin/${BASE_REF_NAME}" + git fetch origin "${BASE_REF_NAME}" + else + # Push to the daily OSS branch: review the accumulated OSS delta. + BASE_REF="origin/main" + git fetch origin main + fi + + CHANGED_FILES="$(git diff --name-only "${BASE_REF}...HEAD")" + + if [ -z "${CHANGED_FILES}" ]; then + echo "No changed files detected." + exit 0 + fi + + echo "Changed files:" + echo "${CHANGED_FILES}" + + BLOCKED_FILES="$( + echo "${CHANGED_FILES}" | grep -E '(^\.github/workflows/|^\.github/actions/|^\.circleci/|^\.cursor/|^Dockerfile$|(^|/)Dockerfile$|^Makefile$|(^|/)Makefile$|(^|/)pyproject\.toml$|(^|/)uv\.lock$|(^|/)poetry\.lock$|(^|/)requirements[^/]*\.txt$|(^|/)package\.json$|(^|/)package-lock\.json$|(^|/)pnpm-lock\.yaml$|(^|/)yarn\.lock$|(^|/)tsconfig[^/]*\.json$|(^|/)(ruff|mypy|pytest|eslint|prettier|vitest|next|vite|jest)\.config\.)' || true + )" + + if [ -n "${BLOCKED_FILES}" ]; then + echo "::error::OSS daily branch contains sensitive workflow, config, dependency, or lockfile changes. Split these into a separate internal/security-reviewed PR." + echo "${BLOCKED_FILES}" + exit 1 + fi + + echo "No sensitive OSS daily file changes detected." + + oss-safe-checks: + name: Run OSS daily safe checks + if: startsWith(github.ref_name, 'litellm_oss_daily_20') || startsWith(github.head_ref, 'litellm_oss_daily_20') || startsWith(github.base_ref, 'litellm_oss_daily_20') + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Run secret scan test + run: | + uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v + + - name: Run Ruff + run: | + uv sync --frozen + cd litellm + uv run --no-sync ruff check . From ad69d6f3f924a7619deab91d7d3d40f391a29854 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:06:56 -0700 Subject: [PATCH 014/101] test: emit e2e coverage lines for loki (#32513) --- tests/e2e/CLAUDE.md | 4 +-- tests/e2e/coverage_registry/README.md | 25 +++++++++++----- tests/e2e/coverage_registry/collector.py | 29 +++++++++++++++---- tests/e2e/coverage_registry/schema.py | 15 ++++++++++ tests/e2e/coverage_registry/test_collector.py | 26 +++++++++++++++++ 5 files changed, 83 insertions(+), 16 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index a4c507ca5ea..502c881c764 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -63,7 +63,7 @@ The harness is fully typed and new code must not add `Any` or widen the basedpyr The set of tests we want is a registry checked into this repo, one row per behavior; that file is the definition of done and the denominator. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the registry against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies -Coverage is organized as module > feature > test. Dashboard modules are Core LLMs, Non-Core LLMs, MCPs, Management/UI, Reliability & Performance, Logging & Guardrails, and Other. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` +Coverage is organized as module > feature > test. Dashboard modules are `Core LLMs`, `Non-Core LLMs`, `MCPs`, `Management/UI`, `Reliability & Performance`, `Logging & Guardrails`, and `Other`. The Loki stdout formatter maps those display modules to log-safe labels (`core_llms`, `non_core_llms`, `mcp`, `management_ui`, `reliability_performance`, `logging_guardrails`, and `other`) without changing JSON or Prometheus labels. A feature is either an endpoint (`/chat/completions`) or a behavior (fallbacks, rate limits; config-driven, with no route of its own). A cell reads like `llm.chat_completions.bedrock_converse.tool_use.stream.works` The metric is coverage: the share of registry rows that have a passing covering test, reported to Grafana per module so a gap surfaces as an uncovered row rather than a silent absence @@ -71,7 +71,7 @@ Tests do not declare a dashboard module directly. They only declare the registry ### Naming grammar per module -LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` are Core LLMs. Other LLM endpoints, including `batches` and `realtime`, roll up as Non-Core LLMs. +LLMs - endpoint features (subject = the route), seeded from the Claude Code compat matrix. `chat_completions`, `messages`, and `responses` roll up to `Core LLMs`. Other LLM endpoints, including `batches` and `realtime`, roll up to `Non-Core LLMs`. ``` llm..... diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md index 4177cba7766..863ce34694d 100644 --- a/tests/e2e/coverage_registry/README.md +++ b/tests/e2e/coverage_registry/README.md @@ -9,18 +9,18 @@ note; the naming grammar lives in `tests/e2e/CLAUDE.md`. A **cell** is one customer-noticeable behavior a single e2e test can assert pass/fail on, for example `llm.chat_completions.bedrock_converse.tool_use.stream.works`. Cells are -grouped `module > feature > test`, with LLM cells split into Core LLMs and Non-Core -LLMs for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a +grouped `module > feature > test`, with LLM cells split into `Core LLMs` and +`Non-Core LLMs` for dashboarding. Each cell carries a tier (P0/P1/P2), a source, and a `fail_before_fix` flag. The rows live in per-prefix YAML files (`llm_*.yaml`, `mgmt.yaml`, `mcp.yaml`, `reliability.yaml`, `logging.yaml`, `guardrail.yaml`, `other.yaml`) and validate against the discriminated union in `schema.py`, so an LLM row cannot carry a guardrail field and vice versa. `llm` rows with `subject_endpoint` of `chat_completions`, `messages`, or -`responses` roll up to "Core LLMs"; all other LLM endpoints roll up to "Non-Core -LLMs". LLM endpoint, route, and capability values are typed in `schema.py`, so new -taxonomy values require an explicit schema change. `logging` and `guardrail` are two -id-prefixes that roll up into the single "Logging & Guardrails" dashboard module. +`responses` roll up to `Core LLMs`; all other LLM endpoints roll up to `Non-Core LLMs`. +LLM endpoint, route, and capability values are typed in `schema.py`, so new taxonomy +values require an explicit schema change. `logging` and `guardrail` are two id-prefixes +that roll up into the single `Logging & Guardrails` dashboard module. A test declares what it covers with a marker: @@ -40,8 +40,17 @@ proxy. Whether a covered cell currently passes or fails is a separate, live conc cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector ``` -Use `--format prometheus` or `--format json` for CI jobs that publish coverage to -Grafana. +Use `--format loki` after the e2e pytest run in the same Kubernetes job/pod to print +structured stdout lines for Loki: + +``` +cd tests/e2e && PYTHONPATH=. python -m coverage_registry.collector --format loki --strict +``` + +This emits exactly one `COVERAGE_TOTAL` line and one `COVERAGE_MODULE` line per module +in `MODULE_ORDER`, in that order. Loki uses log-safe `module=` labels from +`LOKI_MODULE_LABELS` (`core_llms`, `management_ui`, etc.) so existing JSON and +Prometheus consumers keep their human-readable module names unchanged. The headline is overall coverage. The collector also lists markers that point at ids not in the registry, so a typo or an unenumerated behavior surfaces instead of being diff --git a/tests/e2e/coverage_registry/collector.py b/tests/e2e/coverage_registry/collector.py index 3b577106605..f6e59ca4a88 100644 --- a/tests/e2e/coverage_registry/collector.py +++ b/tests/e2e/coverage_registry/collector.py @@ -21,7 +21,7 @@ from pathlib import Path import pytest from .registry import load_registry -from .schema import MODULE_ORDER, Cell, Tier, dashboard_module +from .schema import MODULE_ORDER, Cell, Tier, dashboard_module, loki_module_label E2E_DIR = Path(__file__).resolve().parent.parent @@ -239,13 +239,31 @@ def render_prometheus(report: CoverageReport) -> str: return "\n".join(lines) +def render_loki(report: CoverageReport) -> str: + lines = [ + ( + f"COVERAGE_TOTAL percent={report.coverage_percent:.1f} " + f"covered={report.covered} total={report.total}" + ) + ] + lines.extend( + ( + f"COVERAGE_MODULE module={loki_module_label(module.module)} " + f"percent={module.coverage_percent:.1f} " + f"covered={module.covered} total={module.total}" + ) + for module in report.modules + ) + return "\n".join(lines) + + def main() -> int: parser = ArgumentParser() parser.add_argument( "--format", - choices=("text", "json", "prometheus"), + choices=("text", "json", "prometheus", "loki"), default="text", - help="Output format. Use prometheus or json for Grafana ingestion jobs.", + help="Output format. Use loki for structured stdout lines in the e2e job.", ) parser.add_argument( "--strict", @@ -265,9 +283,8 @@ def main() -> int: "text": render, "json": render_json, "prometheus": render_prometheus, - }[ - args.format - ](report) + "loki": render_loki, + }[args.format](report) print(output) # noqa: T201 # CLI entrypoint output if args.strict and report.orphan_markers: return 1 diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 2e2a00e78ba..bb27fbf0ea0 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -157,6 +157,16 @@ MODULE_ORDER: tuple[str, ...] = ( "Other", ) +LOKI_MODULE_LABELS: dict[str, str] = { + "Core LLMs": "core_llms", + "Non-Core LLMs": "non_core_llms", + "MCPs": "mcp", + "Management/UI": "management_ui", + "Reliability & Performance": "reliability_performance", + "Logging & Guardrails": "logging_guardrails", + "Other": "other", +} + def dashboard_module(cell: Cell) -> str: """Return the Grafana/reporting module for a registry cell.""" @@ -165,3 +175,8 @@ def dashboard_module(cell: Cell) -> str: return "Core LLMs" return "Non-Core LLMs" return PREFIX_ROLLUP[cell.module] + + +def loki_module_label(module: str) -> str: + """Return the log-safe Loki label for a dashboard module.""" + return LOKI_MODULE_LABELS[module] diff --git a/tests/e2e/coverage_registry/test_collector.py b/tests/e2e/coverage_registry/test_collector.py index 355bc52730d..079ee215866 100644 --- a/tests/e2e/coverage_registry/test_collector.py +++ b/tests/e2e/coverage_registry/test_collector.py @@ -15,6 +15,7 @@ from coverage_registry.collector import ( compute_coverage, render, render_json, + render_loki, render_prometheus, ) from coverage_registry.registry import load_registry @@ -24,6 +25,7 @@ from coverage_registry.schema import ( LlmEndpoint, LoggingCell, Tier, + loki_module_label, ) @@ -149,6 +151,30 @@ def test_prometheus_render_exposes_module_coverage_timeseries() -> None: assert "litellm_e2e_coverage_orphan_markers 0" in metrics +def test_loki_render_exposes_exact_stdout_lines_for_loki() -> None: + report = compute_coverage( + (_llm("llm.chat", Tier.P0), _llm("llm.batches", Tier.P0, "batches")), + frozenset({"llm.chat"}), + ) + + lines = render_loki(report).splitlines() + + assert len(lines) == 1 + len(report.modules) + assert lines[0] == "COVERAGE_TOTAL percent=50.0 covered=1 total=2" + assert ( + lines[1] == "COVERAGE_MODULE module=core_llms percent=100.0 covered=1 total=1" + ) + assert ( + lines[2] == "COVERAGE_MODULE module=non_core_llms percent=0.0 covered=0 total=1" + ) + assert [line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:]] == [ + loki_module_label(module.module) for module in report.modules + ] + assert all( + " " not in line.split("module=", 1)[1].split(" ", 1)[0] for line in lines[1:] + ) + + def test_real_registry_loads_and_ids_are_unique() -> None: cells = load_registry() ids = [c.id for c in cells] From 93c047d52eaa665b30931aa4ca9bf7230c0ed74d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:07:58 -0700 Subject: [PATCH 015/101] feat(proxy): make Microsoft Graph endpoint configurable for GCC High (LIT-4282) (#32517) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- litellm/proxy/management_endpoints/ui_sso.py | 22 ++++-- .../proxy/management_endpoints/test_ui_sso.py | 72 +++++++++++++++++++ 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index dbf514d2298..43fdd3ed05a 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -113,7 +113,7 @@ from litellm.proxy.utils import ( from litellm.repositories.table_repositories import SSOConfigRepository from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository -from litellm.secret_managers.main import get_secret_bool, str_to_bool +from litellm.secret_managers.main import get_secret_bool, get_secret_str, str_to_bool from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403 from litellm.types.proxy.management_endpoints.ui_sso import ( DefaultTeamSSOParams, @@ -3737,8 +3737,7 @@ class MicrosoftSSOHandler: Handles Microsoft SSO callback response and returns a CustomOpenID object """ - graph_api_base_url = "https://graph.microsoft.com/v1.0" - graph_api_user_groups_endpoint = f"{graph_api_base_url}/me/memberOf" + DEFAULT_GRAPH_API_BASE_URL = "https://graph.microsoft.com/v1.0" """ Constants @@ -3748,6 +3747,19 @@ class MicrosoftSSOHandler: # used for debugging to show the user groups litellm found from Graph API GRAPH_API_RESPONSE_KEY = "graph_api_user_groups" + @staticmethod + def get_graph_api_base_url() -> str: + """ + Returns the Microsoft Graph API base URL, configurable via the + `MICROSOFT_GRAPH_ENDPOINT` env var so non-default clouds such as Azure + Government (GCC High) can point at `https://graph.microsoft.us/v1.0` + """ + return get_secret_str("MICROSOFT_GRAPH_ENDPOINT") or MicrosoftSSOHandler.DEFAULT_GRAPH_API_BASE_URL + + @staticmethod + def get_graph_api_user_groups_endpoint() -> str: + return f"{MicrosoftSSOHandler.get_graph_api_base_url()}/me/memberOf" + @staticmethod async def get_microsoft_callback_response( request: Request, @@ -3924,7 +3936,7 @@ class MicrosoftSSOHandler: # Fetch user membership from Microsoft Graph API all_group_ids = [] - next_link: Optional[str] = MicrosoftSSOHandler.graph_api_user_groups_endpoint + next_link: Optional[str] = MicrosoftSSOHandler.get_graph_api_user_groups_endpoint() auth_headers = {"Authorization": f"Bearer {access_token}"} page_count = 0 @@ -4007,7 +4019,7 @@ class MicrosoftSSOHandler: Users use Enterprise Applications to manage Groups and Users on Microsoft Entra ID """ - base_url = "https://graph.microsoft.com/v1.0" + base_url = MicrosoftSSOHandler.get_graph_api_base_url() # Endpoint to get app role assignments for the given service principal endpoint = f"/servicePrincipals/{service_principal_id}/appRoleAssignedTo" url = base_url + endpoint diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 32229e3e64e..642f20906a0 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -389,6 +389,78 @@ async def test_get_user_groups_error_handling(): assert len(result) == 0 +@pytest.mark.asyncio +async def test_get_user_groups_uses_default_graph_endpoint(monkeypatch): + monkeypatch.delenv("MICROSOFT_GRAPH_ENDPOINT", raising=False) + + requested_urls: list[str] = [] + + async def mock_get(url, *args, **kwargs): + requested_urls.append(url) + mock = MagicMock() + mock.json.return_value = {"value": []} + return mock + + with patch( + "litellm.proxy.management_endpoints.ui_sso.get_async_httpx_client" + ) as mock_client: + mock_client.return_value = MagicMock() + mock_client.return_value.get = mock_get + + await MicrosoftSSOHandler.get_user_groups_from_graph_api(access_token="mock_token") + + assert requested_urls == ["https://graph.microsoft.com/v1.0/me/memberOf"] + + +@pytest.mark.asyncio +async def test_get_user_groups_uses_configured_graph_endpoint(monkeypatch): + monkeypatch.setenv("MICROSOFT_GRAPH_ENDPOINT", "https://graph.microsoft.us/v1.0") + + requested_urls: list[str] = [] + + async def mock_get(url, *args, **kwargs): + requested_urls.append(url) + mock = MagicMock() + mock.json.return_value = {"value": []} + return mock + + with patch( + "litellm.proxy.management_endpoints.ui_sso.get_async_httpx_client" + ) as mock_client: + mock_client.return_value = MagicMock() + mock_client.return_value.get = mock_get + + await MicrosoftSSOHandler.get_user_groups_from_graph_api(access_token="mock_token") + + assert requested_urls == ["https://graph.microsoft.us/v1.0/me/memberOf"] + + +@pytest.mark.asyncio +async def test_get_group_ids_from_service_principal_uses_configured_graph_endpoint(monkeypatch): + monkeypatch.setenv("MICROSOFT_GRAPH_ENDPOINT", "https://graph.microsoft.us/v1.0") + + requested_urls: list[str] = [] + + async def mock_get(url, *args, **kwargs): + requested_urls.append(url) + mock = MagicMock() + mock.json.return_value = {"value": []} + return mock + + async_client = MagicMock() + async_client.get = mock_get + + await MicrosoftSSOHandler.get_group_ids_from_service_principal( + service_principal_id="sp-123", + async_client=async_client, + access_token="mock_token", + ) + + assert requested_urls == [ + "https://graph.microsoft.us/v1.0/servicePrincipals/sp-123/appRoleAssignedTo" + ] + + def test_get_group_ids_from_graph_api_response(): # Arrange mock_response = MicrosoftGraphAPIUserGroupResponse( From 82fd456b94b36cbfee126e0d30c549b284741a04 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Jul 2026 11:55:59 -0700 Subject: [PATCH 016/101] Revert "ci: skip unit test workflows when only ui or markdown files change (#32422)" This reverts commit 6df5e1b263a77a25a5bb483015fd13a79f3ef410. --- .github/workflows/test-unit-core-utils.yml | 4 ---- .github/workflows/test-unit-documentation.yml | 4 ---- .github/workflows/test-unit-enterprise-routing.yml | 4 ---- .github/workflows/test-unit-integrations.yml | 4 ---- .github/workflows/test-unit-llm-providers.yml | 4 ---- .github/workflows/test-unit-misc.yml | 4 ---- .github/workflows/test-unit-proxy-auth.yml | 4 ---- .github/workflows/test-unit-proxy-db.yml | 4 ---- .github/workflows/test-unit-proxy-endpoints.yml | 4 ---- .github/workflows/test-unit-proxy-infra.yml | 4 ---- .github/workflows/test-unit-proxy-legacy.yml | 4 ---- .github/workflows/test-unit-responses-caching-types.yml | 4 ---- 12 files changed, 48 deletions(-) diff --git a/.github/workflows/test-unit-core-utils.yml b/.github/workflows/test-unit-core-utils.yml index e563679660b..d6d6353238f 100644 --- a/.github/workflows/test-unit-core-utils.yml +++ b/.github/workflows/test-unit-core-utils.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-documentation.yml b/.github/workflows/test-unit-documentation.yml index 2c3d6e46618..4cef791a9b3 100644 --- a/.github/workflows/test-unit-documentation.yml +++ b/.github/workflows/test-unit-documentation.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-enterprise-routing.yml b/.github/workflows/test-unit-enterprise-routing.yml index 7a9b8b00f26..13136c968d1 100644 --- a/.github/workflows/test-unit-enterprise-routing.yml +++ b/.github/workflows/test-unit-enterprise-routing.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-integrations.yml b/.github/workflows/test-unit-integrations.yml index b28ba3456ce..c95ed4e7c24 100644 --- a/.github/workflows/test-unit-integrations.yml +++ b/.github/workflows/test-unit-integrations.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-llm-providers.yml b/.github/workflows/test-unit-llm-providers.yml index fecdcbd3b95..df78564ab0c 100644 --- a/.github/workflows/test-unit-llm-providers.yml +++ b/.github/workflows/test-unit-llm-providers.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-misc.yml b/.github/workflows/test-unit-misc.yml index dbc3bfc8191..7c3b195f0ad 100644 --- a/.github/workflows/test-unit-misc.yml +++ b/.github/workflows/test-unit-misc.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-auth.yml b/.github/workflows/test-unit-proxy-auth.yml index ad534cc0098..97dfaed6e81 100644 --- a/.github/workflows/test-unit-proxy-auth.yml +++ b/.github/workflows/test-unit-proxy-auth.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 35a1a9c78a0..2ac9a3b7c1c 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -5,10 +5,6 @@ on: branches: - main - litellm_internal_staging - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index 7eb3d7719c0..cbb36eebdb9 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" workflow_dispatch: permissions: diff --git a/.github/workflows/test-unit-proxy-infra.yml b/.github/workflows/test-unit-proxy-infra.yml index cb944de5cf9..884d62289b9 100644 --- a/.github/workflows/test-unit-proxy-infra.yml +++ b/.github/workflows/test-unit-proxy-infra.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-proxy-legacy.yml b/.github/workflows/test-unit-proxy-legacy.yml index 9798a4e2277..8db218cd1fc 100644 --- a/.github/workflows/test-unit-proxy-legacy.yml +++ b/.github/workflows/test-unit-proxy-legacy.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read diff --git a/.github/workflows/test-unit-responses-caching-types.yml b/.github/workflows/test-unit-responses-caching-types.yml index 7331544de24..2f177587997 100644 --- a/.github/workflows/test-unit-responses-caching-types.yml +++ b/.github/workflows/test-unit-responses-caching-types.yml @@ -7,10 +7,6 @@ on: - litellm_internal_staging - litellm_oss_staging - "litellm_**" - paths-ignore: - - "ui/**" - - "**.md" - - "**.mdx" permissions: contents: read From c3dccb54cfd0666393e8874cfd007c72de8b33cc Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 8 Jul 2026 12:32:03 -0700 Subject: [PATCH 017/101] fix(health): bridge litellm_metadata into logging object in _batch_health_check (#32520) * fix(health): bridge litellm_metadata into logging object in _batch_health_check * Update litellm/litellm_core_utils/health_check_helpers.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix(health): address review - share single metadata copy, conditional api_base, add tests - Only set api_base in litellm_params when a value actually exists; providers like bedrock/vertex/gemini resolve it implicitly and an empty string overwrites their resolution. - Use a single .copy() for both metadata and litellm_metadata to prevent downstream drift between the two references. - Add 6 unit tests covering metadata bridging, api_base omission, guard conditions, and dispatch routing. Signed-off-by: pramod * refactor(health): use update_from_kwargs helper for metadata bridge Collapses the manual metadata/litellm_metadata plumbing in _batch_health_check into a single update_from_kwargs call, matching how the sibling batch/image/rerank/ocr surfaces bridge metadata onto the pre-injected logging object. Drops the bare Dict typing and the inline comment, and switches the tests to assert against the helper. --------- Signed-off-by: pramod Co-authored-by: pramod Co-authored-by: Pramod B <155433727+BPRMD18@users.noreply.github.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../health_check_helpers.py | 11 ++ .../test_health_check_helpers.py | 137 ++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 405366382a1..42ac82abf8b 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -95,6 +95,17 @@ class HealthCheckHelpers: """ import litellm + logging_obj = filtered_model_params.get("litellm_logging_obj") + if logging_obj is not None: + api_base = filtered_model_params.get("api_base") + logging_obj.update_from_kwargs( + kwargs=filtered_model_params, + model=filtered_model_params.get("model"), + user=None, + optional_params={}, + litellm_params={"api_base": api_base} if api_base else None, + ) + if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS: return await litellm.alist_batches(**filtered_model_params) else: diff --git a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py index 02d72c89e80..e8ef8f15142 100644 --- a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py @@ -14,6 +14,7 @@ from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth +from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS def test_update_model_params_with_health_check_tracking_information(): @@ -140,3 +141,139 @@ async def test_ahealth_check_failure_masks_raw_request_headers(): assert headers["Content-Type"] == "application/json" print(f"Masked Authorization header: {headers.get('Authorization', 'NOT FOUND')}") + + +@pytest.mark.asyncio +async def test_batch_health_check_bridges_metadata_into_logging_obj(): + """_batch_health_check must call update_from_kwargs on the pre-injected + logging object so callbacks receive identity/tracking fields in + model_call_details["litellm_params"]["metadata"].""" + mock_logging_obj = MagicMock() + mock_logging_obj.update_from_kwargs = MagicMock() + + litellm_metadata = { + "tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME], + "user_api_key_alias": "health-check-key", + } + + filtered_model_params = { + "model": "openai/gpt-4", + "api_base": "https://api.openai.com", + "litellm_logging_obj": mock_logging_obj, + "litellm_metadata": litellm_metadata, + } + + with patch("litellm.alist_batches", new_callable=AsyncMock, return_value={}): + await HealthCheckHelpers._batch_health_check( + custom_llm_provider="openai", + model_params={"model": "openai/gpt-4"}, + filtered_model_params=filtered_model_params, + ) + + mock_logging_obj.update_from_kwargs.assert_called_once() + call_kwargs = mock_logging_obj.update_from_kwargs.call_args[1] + assert call_kwargs["model"] == "openai/gpt-4" + assert call_kwargs["kwargs"] is filtered_model_params + assert call_kwargs["litellm_params"] == {"api_base": "https://api.openai.com"} + + +@pytest.mark.asyncio +async def test_batch_health_check_omits_api_base_when_absent(): + """api_base must not appear in litellm_params when the provider resolves + it implicitly (bedrock, vertex, gemini).""" + mock_logging_obj = MagicMock() + mock_logging_obj.update_from_kwargs = MagicMock() + + litellm_metadata = {"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME]} + + filtered_model_params = { + "model": "bedrock/anthropic.claude-v2", + "litellm_logging_obj": mock_logging_obj, + "litellm_metadata": litellm_metadata, + } + + with patch("litellm.acompletion", new_callable=AsyncMock, return_value={}): + await HealthCheckHelpers._batch_health_check( + custom_llm_provider="bedrock", + model_params={"model": "bedrock/anthropic.claude-v2"}, + filtered_model_params=filtered_model_params, + ) + + call_kwargs = mock_logging_obj.update_from_kwargs.call_args[1] + assert call_kwargs["litellm_params"] is None + + +@pytest.mark.asyncio +async def test_batch_health_check_skips_bridge_when_no_logging_obj(): + """When litellm_logging_obj is absent, dispatch still proceeds.""" + litellm_metadata = {"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME]} + + filtered_model_params = { + "model": "openai/gpt-4", + "litellm_metadata": litellm_metadata, + } + + with patch( + "litellm.alist_batches", new_callable=AsyncMock, return_value={} + ) as mock_alist: + await HealthCheckHelpers._batch_health_check( + custom_llm_provider="openai", + model_params={"model": "openai/gpt-4"}, + filtered_model_params=filtered_model_params, + ) + mock_alist.assert_called_once() + + +@pytest.mark.asyncio +async def test_batch_health_check_uses_alist_batches_for_supported_providers(): + """Providers in LIST_BATCHES_SUPPORTED_PROVIDERS dispatch to alist_batches.""" + mock_logging_obj = MagicMock() + mock_logging_obj.update_from_kwargs = MagicMock() + + litellm_metadata = {"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME]} + + for provider in LIST_BATCHES_SUPPORTED_PROVIDERS: + filtered_model_params = { + "model": f"{provider}/some-model", + "litellm_logging_obj": mock_logging_obj, + "litellm_metadata": litellm_metadata, + } + + with patch( + "litellm.alist_batches", new_callable=AsyncMock, return_value={} + ) as mock_alist: + await HealthCheckHelpers._batch_health_check( + custom_llm_provider=provider, + model_params={"model": f"{provider}/some-model"}, + filtered_model_params=filtered_model_params, + ) + mock_alist.assert_called_once() + + +@pytest.mark.asyncio +async def test_batch_health_check_falls_back_to_acompletion_for_unsupported(): + """Providers not in LIST_BATCHES_SUPPORTED_PROVIDERS fall back to acompletion.""" + mock_logging_obj = MagicMock() + mock_logging_obj.update_from_kwargs = MagicMock() + + litellm_metadata = {"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME]} + + filtered_model_params = { + "model": "bedrock/anthropic.claude-v2", + "litellm_logging_obj": mock_logging_obj, + "litellm_metadata": litellm_metadata, + } + + model_params = {"model": "bedrock/anthropic.claude-v2", "messages": []} + + with ( + patch("litellm.alist_batches", new_callable=AsyncMock) as mock_alist, + patch("litellm.acompletion", new_callable=AsyncMock, return_value={}) as mock_acompletion, + ): + await HealthCheckHelpers._batch_health_check( + custom_llm_provider="bedrock", + model_params=model_params, + filtered_model_params=filtered_model_params, + ) + mock_alist.assert_not_called() + mock_acompletion.assert_called_once_with(**model_params) From 12d1873b44286f1bb9c1e07970958b5e134354b1 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Jul 2026 13:23:34 -0700 Subject: [PATCH 018/101] feat(ui): add enterprise license expiry banner to admin dashboard Surfaces a persistent, tiered banner under the dashboard navbar when an airgapped enterprise license is close to expiring: an amber, session-dismissible warning within 30 days, a non-dismissible red alert within 7 days, and a non-dismissible red banner once the date has passed. It reads the existing /health/license endpoint, so no backend change is needed, and is driven strictly by expiration_date; community and remote-validated instances that report no date show nothing. Shared day-count math is extracted to licenseUtils so the banner and the existing UsageIndicator widget stay in sync --- .../hooks/license/useLicenseInfo.ts | 15 +++ .../src/app/(dashboard)/layout.tsx | 2 + .../components/LicenseExpiryBanner.test.tsx | 90 ++++++++++++++++++ .../src/components/LicenseExpiryBanner.tsx | 92 +++++++++++++++++++ .../src/components/UsageIndicator.tsx | 12 +-- .../src/utils/licenseUtils.test.ts | 61 ++++++++++++ .../src/utils/licenseUtils.ts | 49 ++++++++++ 7 files changed, 310 insertions(+), 11 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts create mode 100644 ui/litellm-dashboard/src/components/LicenseExpiryBanner.test.tsx create mode 100644 ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx create mode 100644 ui/litellm-dashboard/src/utils/licenseUtils.test.ts create mode 100644 ui/litellm-dashboard/src/utils/licenseUtils.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts new file mode 100644 index 00000000000..f4574c36ef6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts @@ -0,0 +1,15 @@ +import { useQuery, UseQueryResult } from "@tanstack/react-query"; +import { getLicenseInfo, LicenseInfo } from "@/components/networking"; +import { createQueryKeys } from "../common/queryKeysFactory"; + +const licenseInfoKeys = createQueryKeys("licenseInfo"); + +export const useLicenseInfo = (accessToken: string | null | undefined): UseQueryResult => { + return useQuery({ + queryKey: licenseInfoKeys.detail("license"), + queryFn: () => getLicenseInfo(accessToken!), + enabled: Boolean(accessToken), + staleTime: 5 * 60 * 1000, + retry: false, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 09951dc1923..c84209b80cf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -8,6 +8,7 @@ import { useAuth } from "@/contexts/AuthContext"; import SidebarProvider from "@/app/(dashboard)/components/SidebarProvider"; import { useRouter, useSearchParams, usePathname } from "next/navigation"; import { DebugWarningBanner } from "@/components/DebugWarningBanner"; +import { LicenseExpiryBanner } from "@/components/LicenseExpiryBanner"; import { MIGRATED_PAGES, migratedHref, legacyPageHref, legacyKeyForPathname } from "@/utils/migratedPages"; import { PluginModeProvider, usePluginMode } from "@/contexts/PluginModeContext"; import { createApiClient } from "@/lib/http/client"; @@ -116,6 +117,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { onToggleSidebar={() => setSidebarCollapsed((v) => !v)} /> +
{mode !== "ai-gateway" ? (
diff --git a/ui/litellm-dashboard/src/components/LicenseExpiryBanner.test.tsx b/ui/litellm-dashboard/src/components/LicenseExpiryBanner.test.tsx new file mode 100644 index 00000000000..d6b419ace7c --- /dev/null +++ b/ui/litellm-dashboard/src/components/LicenseExpiryBanner.test.tsx @@ -0,0 +1,90 @@ +import React from "react"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { render, screen, fireEvent } from "@testing-library/react"; +import { LicenseExpiryBannerView } from "./LicenseExpiryBanner"; +import { LicenseInfo } from "./networking"; + +const daysFromNow = (n: number): string => { + const date = new Date(); + date.setUTCDate(date.getUTCDate() + n); + return date.toISOString().slice(0, 10); +}; + +const licenseWith = (expiration_date: string | null): LicenseInfo => ({ + has_license: expiration_date !== null, + license_type: expiration_date !== null ? "enterprise" : "community", + expiration_date, + allowed_features: [], + limits: { max_users: null, max_teams: null }, +}); + +describe("LicenseExpiryBannerView", () => { + beforeEach(() => { + sessionStorage.clear(); + }); + + afterEach(() => { + sessionStorage.clear(); + }); + + it("renders nothing when there is no license info", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("renders nothing when expiration_date is null (community or remote-validated)", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("renders nothing when expiry is more than 30 days out", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("shows a dismissible amber warning within 30 days", () => { + const { container } = render(); + expect(screen.getByText(/expires in 20 days/)).toBeInTheDocument(); + expect(container.querySelector(".ant-alert-warning")).toBeInTheDocument(); + expect(screen.queryByRole("button")).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "sales@berri.ai" })).toHaveAttribute("href", "mailto:sales@berri.ai"); + }); + + it("shows a non-dismissible red critical alert within 7 days", () => { + const { container } = render(); + expect(screen.getByText(/expires in 5 days/)).toBeInTheDocument(); + expect(container.querySelector(".ant-alert-error")).toBeInTheDocument(); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("says 'expires today' on the expiration day", () => { + render(); + expect(screen.getByText(/expires today/)).toBeInTheDocument(); + }); + + it("shows a non-dismissible red expired alert stating features are disabled", () => { + const { container } = render(); + expect(screen.getByText(/expired on/)).toBeInTheDocument(); + expect(screen.getByText(/features are now disabled/i)).toBeInTheDocument(); + expect(container.querySelector(".ant-alert-error")).toBeInTheDocument(); + expect(screen.queryByRole("button")).not.toBeInTheDocument(); + }); + + it("hides the warning after dismissal and stays hidden within the session", () => { + const expiration = daysFromNow(20); + const { unmount } = render(); + fireEvent.click(screen.getByRole("button")); + expect(screen.queryByText(/expires in 20 days/)).not.toBeInTheDocument(); + + unmount(); + render(); + expect(screen.queryByText(/expires in 20 days/)).not.toBeInTheDocument(); + }); + + it("still shows a critical alert even when its date was previously dismissed", () => { + const expiration = daysFromNow(5); + sessionStorage.setItem(`litellm:licenseExpiryBannerDismissed:${expiration}`, "true"); + render(); + expect(screen.getByText(/expires in 5 days/)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx b/ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx new file mode 100644 index 00000000000..e5b8a65168a --- /dev/null +++ b/ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx @@ -0,0 +1,92 @@ +"use client"; + +import React, { useState } from "react"; +import { Alert } from "antd"; +import { LicenseInfo } from "@/components/networking"; +import { useLicenseInfo } from "@/app/(dashboard)/hooks/license/useLicenseInfo"; +import { formatExpiryDate, getDaysUntilExpiration, getLicenseExpiryTier } from "@/utils/licenseUtils"; + +const DISMISS_KEY_PREFIX = "litellm:licenseExpiryBannerDismissed:"; +const SALES_EMAIL = "sales@berri.ai"; + +const salesLink = {SALES_EMAIL}; + +interface LicenseExpiryBannerProps { + accessToken: string | null; +} + +interface LicenseExpiryBannerViewProps { + licenseInfo: LicenseInfo | null; +} + +const describeCountdown = (days: number): string => { + if (days <= 0) { + return "expires today"; + } + if (days === 1) { + return "expires in 1 day"; + } + return `expires in ${days} days`; +}; + +export const LicenseExpiryBannerView: React.FC = ({ licenseInfo }) => { + const [locallyDismissed, setLocallyDismissed] = useState(false); + + const expirationDate = licenseInfo?.expiration_date ?? null; + const tier = getLicenseExpiryTier(expirationDate); + const days = getDaysUntilExpiration(expirationDate); + + if (expirationDate === null || tier === "none" || days === null) { + return null; + } + + const isDismissible = tier === "warning"; + const dismissKey = `${DISMISS_KEY_PREFIX}${expirationDate}`; + const previouslyDismissed = + isDismissible && typeof window !== "undefined" ? sessionStorage.getItem(dismissKey) === "true" : false; + + if (isDismissible && (locallyDismissed || previouslyDismissed)) { + return null; + } + + const formattedDate = formatExpiryDate(expirationDate); + + const message = + tier === "expired" + ? `Your LiteLLM Enterprise license expired on ${formattedDate}` + : `Your LiteLLM Enterprise license ${describeCountdown(days)} (${formattedDate})`; + + const description = + tier === "expired" ? ( + <>Enterprise features are now disabled. Reach out to {salesLink} to restore access + ) : tier === "critical" ? ( + <>Renew now to avoid losing enterprise features. Reach out to {salesLink} + ) : ( + <>Renew before it lapses to keep enterprise features. Reach out to {salesLink} + ); + + const handleClose = () => { + if (typeof window !== "undefined") { + sessionStorage.setItem(dismissKey, "true"); + } + setLocallyDismissed(true); + }; + + return ( + + ); +}; + +export const LicenseExpiryBanner: React.FC = ({ accessToken }) => { + const { data } = useLicenseInfo(accessToken); + return ; +}; diff --git a/ui/litellm-dashboard/src/components/UsageIndicator.tsx b/ui/litellm-dashboard/src/components/UsageIndicator.tsx index 030df228de0..8165f8eb6f6 100644 --- a/ui/litellm-dashboard/src/components/UsageIndicator.tsx +++ b/ui/litellm-dashboard/src/components/UsageIndicator.tsx @@ -15,6 +15,7 @@ import { useEffect, useState } from "react"; import { getRemainingUsers, getLicenseInfo, LicenseInfo } from "./networking"; import { cn } from "@/lib/cva.config"; +import { getDaysUntilExpiration } from "@/utils/licenseUtils"; interface UsageIndicatorProps { accessToken: string | null; @@ -30,17 +31,6 @@ interface UsageData { total_teams_remaining: number | null; } -// Calculate days until expiration -const getDaysUntilExpiration = (expirationDate: string | null): number | null => { - if (!expirationDate) return null; - const expDate = new Date(expirationDate + "T00:00:00Z"); // Force UTC midnight - const now = new Date(); - now.setHours(0, 0, 0, 0); // Normalize to local midnight - const diffTime = expDate.getTime() - now.getTime(); - const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)); - return diffDays; -}; - // Format expiration for display const formatExpirationDisplay = (daysRemaining: number | null): string => { if (daysRemaining === null) return "No expiration"; diff --git a/ui/litellm-dashboard/src/utils/licenseUtils.test.ts b/ui/litellm-dashboard/src/utils/licenseUtils.test.ts new file mode 100644 index 00000000000..717b8f0d90d --- /dev/null +++ b/ui/litellm-dashboard/src/utils/licenseUtils.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect } from "vitest"; +import { type LicenseExpiryTier, formatExpiryDate, getDaysUntilExpiration, getLicenseExpiryTier } from "./licenseUtils"; + +const NOW = new Date("2026-07-08T00:00:00Z"); + +describe("getDaysUntilExpiration", () => { + it("returns null for a null expiration", () => { + expect(getDaysUntilExpiration(null, NOW)).toBeNull(); + }); + + it("returns null for an unparseable date", () => { + expect(getDaysUntilExpiration("not-a-date", NOW)).toBeNull(); + }); + + it("returns 0 for an expiration on the current UTC day", () => { + expect(getDaysUntilExpiration("2026-07-08", NOW)).toBe(0); + }); + + it("returns a positive count for future dates", () => { + expect(getDaysUntilExpiration("2026-07-15", NOW)).toBe(7); + expect(getDaysUntilExpiration("2026-08-07", NOW)).toBe(30); + }); + + it("returns a negative count for a past date", () => { + expect(getDaysUntilExpiration("2026-07-07", NOW)).toBe(-1); + }); + + it("is timezone-independent within a UTC day", () => { + expect(getDaysUntilExpiration("2026-08-07", new Date("2026-07-08T00:00:01Z"))).toBe(30); + expect(getDaysUntilExpiration("2026-08-07", new Date("2026-07-08T23:59:59Z"))).toBe(30); + }); +}); + +describe("getLicenseExpiryTier", () => { + const cases: Array<[string | null, LicenseExpiryTier]> = [ + [null, "none"], + ["not-a-date", "none"], + ["2026-08-08", "none"], + ["2026-08-07", "warning"], + ["2026-07-16", "warning"], + ["2026-07-15", "critical"], + ["2026-07-09", "critical"], + ["2026-07-08", "critical"], + ["2026-07-07", "expired"], + ["2026-01-01", "expired"], + ]; + + it.each(cases)("classifies %s as %s", (date, expected) => { + expect(getLicenseExpiryTier(date, NOW)).toBe(expected); + }); +}); + +describe("formatExpiryDate", () => { + it("formats an ISO date as a human-readable UTC date", () => { + expect(formatExpiryDate("2026-07-31")).toBe("Jul 31, 2026"); + }); + + it("returns the input unchanged when unparseable", () => { + expect(formatExpiryDate("bogus")).toBe("bogus"); + }); +}); diff --git a/ui/litellm-dashboard/src/utils/licenseUtils.ts b/ui/litellm-dashboard/src/utils/licenseUtils.ts new file mode 100644 index 00000000000..b2681664c56 --- /dev/null +++ b/ui/litellm-dashboard/src/utils/licenseUtils.ts @@ -0,0 +1,49 @@ +export type LicenseExpiryTier = "none" | "warning" | "critical" | "expired"; + +export const LICENSE_EXPIRY_WARNING_DAYS = 30; +export const LICENSE_EXPIRY_CRITICAL_DAYS = 7; + +export const getDaysUntilExpiration = (expirationDate: string | null, now: Date = new Date()): number | null => { + if (!expirationDate) { + return null; + } + + const expiration = new Date(`${expirationDate}T00:00:00Z`); + if (Number.isNaN(expiration.getTime())) { + return null; + } + + const nowUtcMidnight = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()); + const diffMs = expiration.getTime() - nowUtcMidnight; + return Math.ceil(diffMs / (1000 * 60 * 60 * 24)); +}; + +export const getLicenseExpiryTier = (expirationDate: string | null, now: Date = new Date()): LicenseExpiryTier => { + const days = getDaysUntilExpiration(expirationDate, now); + if (days === null) { + return "none"; + } + if (days < 0) { + return "expired"; + } + if (days <= LICENSE_EXPIRY_CRITICAL_DAYS) { + return "critical"; + } + if (days <= LICENSE_EXPIRY_WARNING_DAYS) { + return "warning"; + } + return "none"; +}; + +export const formatExpiryDate = (expirationDate: string): string => { + const date = new Date(`${expirationDate}T00:00:00Z`); + if (Number.isNaN(date.getTime())) { + return expirationDate; + } + return date.toLocaleDateString("en-US", { + year: "numeric", + month: "short", + day: "numeric", + timeZone: "UTC", + }); +}; From 85d1fe6e2a535e9edfc1ae0b0854eb204573c7ba Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 8 Jul 2026 13:44:48 -0700 Subject: [PATCH 019/101] fix(otel): restore error.* span attributes on v2 error spans (LIT-4179) (#32524) The v2 emitter has never stamped error.message / error.code / error.stack_trace / error.llm_provider as span attributes; only error.type reached the wire. Backends that flatten span attributes into label indexes (Elastic APM labels.error_*, Datadog span tags) lost these four fields when v2 became the active integration on v1.90+ for otel_v2-flagged deployments. The pre-existing exception span event carrying the full message (LIT-3758) is unchanged; the message now rides both places at once, matching v1s shape. SpanError grows three optional detail fields; _parse_error threads them from StandardLoggingPayloadErrorInformation; the emitters error branch stamps them via a new module-level helper, guarded per field so guardrail-shape errors are not polluted with empty attributes. New semconv constants mirror open_inference.ErrorAttributes byte-for-byte, so v1 and v2 consumers read the same keys. Regression tests extend the mapped test files under tests/test_litellm/integrations/otel/. pytest reports 243 passed. --- litellm/integrations/otel/__init__.py | 2 + litellm/integrations/otel/emitter.py | 35 +++++- litellm/integrations/otel/model/payloads.py | 6 + litellm/integrations/otel/model/semconv.py | 20 ++++ .../otel/test_otel_v2_components.py | 110 ++++++++++++++++-- .../otel/test_otel_v2_sources_of_truth.py | 47 +++++++- 6 files changed, 203 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/otel/__init__.py b/litellm/integrations/otel/__init__.py index 7f78f7156b4..5e167e006ff 100644 --- a/litellm/integrations/otel/__init__.py +++ b/litellm/integrations/otel/__init__.py @@ -52,6 +52,7 @@ from litellm.integrations.otel.model.semconv import ( GenAIProvider, JsonRpc, LiteLLM, + LiteLLMError, MCPMethod, Metric, Network, @@ -87,6 +88,7 @@ __all__ = [ "HTTP", "JsonRpc", "LiteLLM", + "LiteLLMError", "MCP", "MCPMethod", "Metric", diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 8441cbae834..46aa166a8bb 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -16,9 +16,10 @@ from litellm.integrations.otel.model.payloads import ( MCPListToolsSpanData, MCPToolCallSpanData, ServiceSpanData, + SpanError, ) from litellm.integrations.otel.plumbing.providers import to_otel_span_kind -from litellm.integrations.otel.model.semconv import Error, ExceptionEvent +from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError from litellm.integrations.otel.model.spans import ( SPAN_REGISTRY, SpanRole, @@ -49,6 +50,27 @@ _NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = { _DEDUP_CACHE_MAX = 10_000 +def _stamp_otel_error_attributes(span: Span, error_type: str, resolved_message: str) -> None: + """Stamp the OTel-semconv error attributes (``error.type`` + ``error.message``). + ``error_type`` and ``resolved_message`` are ``finish_span``'s already-computed + fallback chains, so the pair on the status, event, and attributes stays in + lockstep.""" + span.set_attribute(Error.TYPE, error_type) + span.set_attribute(Error.MESSAGE, resolved_message) + + +def _stamp_litellm_error_attributes(span: Span, error: SpanError) -> None: + """Stamp litellm-specific error detail attributes. Emitted only when the + corresponding field is populated so guardrail-shape errors carrying only a + message aren't polluted with empty detail keys.""" + if error.code: + span.set_attribute(LiteLLMError.CODE, error.code) + if error.stack_trace: + span.set_attribute(LiteLLMError.STACK_TRACE, error.stack_trace) + if error.llm_provider: + span.set_attribute(LiteLLMError.LLM_PROVIDER, error.llm_provider) + + class SpanEmitter: def __init__( self, @@ -190,12 +212,13 @@ class SpanEmitter: if error and (error.error_type or error.message): error_type = error.error_type or "error" message = error.message or error.error_type or "error" - span.set_attribute(Error.TYPE, error_type) + _stamp_otel_error_attributes(span, error_type, message) + _stamp_litellm_error_attributes(span, error) span.set_status(Status(StatusCode.ERROR, message)) - # Carry the full message on the standard ``exception`` event so backends - # map it as full text under ``exception.message``. Setting it as a bare - # string attribute instead lets backends like Elasticsearch dynamic-map - # it to a ``keyword`` capped at 1024 chars, truncating the message. + # Also emit the semconv ``exception`` event so backends that + # dynamic-map unknown string span attrs to ``keyword`` (e.g. + # Elasticsearch with a 1024-char ``ignore_above``) still see the + # full untruncated message on the recognized event field. span.add_event( ExceptionEvent.NAME, {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message}, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index fcd710492f0..4a8f01858b5 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -141,6 +141,9 @@ class LLMCost: class SpanError: error_type: str | None = None message: str | None = None + code: str | None = None + stack_trace: str | None = None + llm_provider: str | None = None @dataclass(frozen=True) @@ -571,6 +574,9 @@ def _parse_error(payload: "StandardLoggingPayload") -> SpanError | None: return SpanError( error_type=as_str(info.get("error_class")) or as_str(info.get("error_code")), message=as_str(info.get("error_message")) or as_str(payload.get("error_str")), + code=as_str(info.get("error_code")), + stack_trace=as_str(info.get("traceback")), + llm_provider=as_str(info.get("llm_provider")), ) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 4e725ae0a29..69d1e454655 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -144,7 +144,27 @@ class Client: class Error: + """OTel-defined error attribute keys, from the semconv ``error.*`` registry. + ``MESSAGE`` is marked *Deprecated* upstream in favor of domain-specific + error message keys plus ``exception.message`` on the exception event, but + is still defined and stamped by litellm's v1 integration; keeping it here + for byte-for-byte parity.""" + TYPE: Final = "error.type" + MESSAGE: Final = "error.message" + + +class LiteLLMError: + """LiteLLM-specific error attribute keys. Emitted under the ``error.*`` + namespace (not ``litellm.*``) for byte-for-byte compat with the v1 + integration in ``opentelemetry.py``; consumers reading these keys on v1 + spans read the same keys on v2 spans. OTel semconv does not define any of + these three, and per its extension rules a namespace may carry additional + vendor keys as long as they don't collide with defined names.""" + + CODE: Final = "error.code" + STACK_TRACE: Final = "error.stack_trace" + LLM_PROVIDER: Final = "error.llm_provider" class ExceptionEvent: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 19eef284b91..298047ec18b 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -579,13 +579,10 @@ def _exception_event(span): def test_error_message_recorded_as_full_exception_event_untruncated(): - """Regression for the Elasticsearch keyword/ignore_above:1024 truncation. - - A long error message must survive intact on the standard ``exception`` - event under ``exception.message`` — not get dropped onto a bare string - attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK - must not truncate it either, so a 5000-char message stays 5000 chars. - """ + """The ``exception`` event carries the full untruncated message under + ``exception.message`` so backends that dynamic-map unknown string span + attrs to ``keyword`` (e.g. Elasticsearch with a 1024-char ``ignore_above``) + still see it in full via the semconv-recognized event field.""" from litellm.integrations.otel.model.semconv import Error, ExceptionEvent long_message = "boom: " + "x" * 5000 @@ -596,13 +593,108 @@ def test_error_message_recorded_as_full_exception_event_untruncated(): assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024 assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError" - # error.type stays a low-cardinality attribute; the message does NOT become a - # bare string attribute (which is what got truncated). + # error.type stays a low-cardinality attribute; the exception EVENT field + # ``exception.message`` never becomes a bare string attribute. assert span.attributes[Error.TYPE] == "litellm.APIError" assert ExceptionEvent.MESSAGE not in span.attributes assert span.status.description == long_message +def test_error_details_stamped_as_span_attributes_for_labels_ingest(): + """OTel-defined keys and litellm-specific detail keys both ride span + attributes so backends that flatten attrs into label indexes (Elastic APM + ``labels.*``, Datadog span tags) render them. The exception event with the + full untruncated message stays alongside — both places, matching v1's + shape.""" + from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError + from litellm.integrations.otel.emitter import SpanEmitter + + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider, exporter = providers.in_memory_provider(cfg) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg) + data = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model=None, + response_id=None, + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=(), + error=SpanError( + error_type="litellm.BadRequestError", + message="400: violated moderation policy", + code="400", + stack_trace="File proxy_server.py line 8570 ...", + llm_provider="openai", + ), + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + engine.emit(SpanRole.LLM_CALL, data) + (span,) = exporter.get_finished_spans() + + # OTel-defined keys (from the ``error.*`` semconv registry). + assert span.attributes[Error.TYPE] == "litellm.BadRequestError" + assert span.attributes[Error.MESSAGE] == "400: violated moderation policy" + # LiteLLM-specific detail keys — vendor-namespaced under ``error.*`` + # for v1-parity, not defined by OTel semconv. + assert span.attributes[LiteLLMError.CODE] == "400" + assert span.attributes[LiteLLMError.STACK_TRACE] == "File proxy_server.py line 8570 ..." + assert span.attributes[LiteLLMError.LLM_PROVIDER] == "openai" + + # The exception event carries the same message on the span too. + event = _exception_event(span) + assert event.attributes[ExceptionEvent.MESSAGE] == "400: violated moderation policy" + + +def test_error_details_omitted_when_span_error_carries_only_message(): + """A guardrail-shape error (message only, no code/traceback/provider) must + not pollute the span with empty-string detail attributes. Only the keys + that carry real data land.""" + from litellm.integrations.otel.model.semconv import Error, LiteLLMError + + span = _emit_error_span("guardrail rejected", error_type="ContentFilter") + + assert span.attributes[Error.TYPE] == "ContentFilter" + assert span.attributes[Error.MESSAGE] == "guardrail rejected" + # LiteLLM-specific detail keys aren't stamped when the SpanError doesn't + # carry them. + assert LiteLLMError.CODE not in span.attributes + assert LiteLLMError.STACK_TRACE not in span.attributes + assert LiteLLMError.LLM_PROVIDER not in span.attributes + + +def test_v2_error_attribute_keys_match_v1_error_attributes_byte_for_byte(): + """v1 (``opentelemetry.py``) and v2 (``otel/`` package) stamp identical + span-attribute keys so consumers reading ``labels.error_message`` don't + care which integration produced the span. Renaming either side is a + breaking change for downstream dashboards; this test locks the vocabulary.""" + from litellm.integrations._types.open_inference import ErrorAttributes + from litellm.integrations.otel.model.semconv import Error, LiteLLMError + + assert Error.TYPE == ErrorAttributes.ERROR_TYPE + assert Error.MESSAGE == ErrorAttributes.ERROR_MESSAGE + assert LiteLLMError.CODE == ErrorAttributes.ERROR_CODE + assert LiteLLMError.STACK_TRACE == ErrorAttributes.ERROR_STACK_TRACE + assert LiteLLMError.LLM_PROVIDER == ErrorAttributes.ERROR_LLM_PROVIDER + + +def test_error_message_falls_back_to_error_type_when_message_absent(): + """A ``SpanError(error_type=..., message=None)`` still renders on the span: + the resolved message is the error_type, and it lands on ``error.message``, + the exception event, and the span-status description in lockstep so a + single-source-of-truth view isn't inconsistent.""" + from litellm.integrations.otel.model.semconv import Error, ExceptionEvent + + span = _emit_error_span(message=None, error_type="RateLimitError") + + assert span.attributes[Error.MESSAGE] == "RateLimitError" + assert _exception_event(span).attributes[ExceptionEvent.MESSAGE] == "RateLimitError" + assert span.status.description == "RateLimitError" + + def test_success_span_records_no_exception_event(): from litellm.integrations.otel.emitter import SpanEmitter from litellm.integrations.otel.model.semconv import ExceptionEvent diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 834a484090f..89aa73a6066 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -144,11 +144,13 @@ def _all_constants(cls): def test_attribute_keys_are_unique_across_namespaces(): - from litellm.integrations.otel import MCP, Client, JsonRpc, Network + from litellm.integrations.otel import MCP, Client, JsonRpc, LiteLLMError, Network # prefixes are allowed to be substrings; exact keys must not collide. + # ``LiteLLMError`` shares the ``error.*`` prefix with ``Error`` by design + # (v1-parity); the assert below is the guarantee they never overlap. exact = set() - for cls in (GenAI, Error, Server, HTTP, DB, MCP, JsonRpc, Network, Client): + for cls in (GenAI, Error, LiteLLMError, Server, HTTP, DB, MCP, JsonRpc, Network, Client): for key in _all_constants(cls): assert key not in exact, f"duplicate attribute key {key}" exact.add(key) @@ -342,6 +344,47 @@ def test_llm_call_adapter_failure_path(): assert data.error.message == "429 slow down" +def test_llm_call_adapter_carries_error_detail_fields(): + """``_parse_error`` threads the full detail set from ``error_information`` + (``error_code``, ``traceback``, ``llm_provider``) onto ``SpanError`` so the + emitter can stamp them as span attributes.""" + payload = _sample_payload( + status="failure", + error_information={ + "error_class": "BadRequestError", + "error_message": "400 violated moderation policy", + "error_code": "400", + "traceback": "File proxy_server.py line 8570 ...", + "llm_provider": "openai", + }, + ) + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.error is not None + assert data.error.error_type == "BadRequestError" + assert data.error.message == "400 violated moderation policy" + assert data.error.code == "400" + assert data.error.stack_trace == "File proxy_server.py line 8570 ..." + assert data.error.llm_provider == "openai" + + +def test_llm_call_adapter_error_details_default_to_none_when_absent(): + """Guardrail-shape payloads carry only ``error_class`` + ``error_message``. + The detail fields must stay ``None`` so the emitter's ``if error.code:`` + guards skip stamping empty attributes.""" + payload = _sample_payload( + status="failure", + error_information={ + "error_class": "ContentFilter", + "error_message": "guardrail rejected", + }, + ) + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.error is not None + assert data.error.code is None + assert data.error.stack_trace is None + assert data.error.llm_provider is None + + def test_adapter_is_resilient_to_minimal_payload(): data = LLMCallSpanData.from_standard_logging_payload({}) assert data.request_model == "" From e9e30dffb68264e497d846763853e4f1c96939e7 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 8 Jul 2026 13:47:53 -0700 Subject: [PATCH 020/101] refactor(ui): reskin shared DataTable from tremor onto shadcn table primitives (#32209) * test(ui): characterize DataTable behavior before shadcn reskin Pins the shared view_logs DataTable contract with library-agnostic queries ahead of the tremor-to-shadcn table migration: loading and empty states, TanStack column defs with custom cell renderers, onRowClick payload, both expansion render paths (colspan sub-component and sibling child rows), the getRowCanExpand gate, and client-side sorting on and off. These must pass unchanged after the reskin. * refactor(ui): reskin shared DataTable from tremor onto shadcn table primitives Swaps the view_logs DataTable's presentational layer from @tremor/react to the in-repo components/ui/table primitives and hardens the seam that every later table migration copies: - getRowId is injected instead of hardcoded to request_id through an any cast; identity defaults to the row index and the logs page now passes request_id explicitly, keeping expansion state attached to the right row across refetch reorders - one expansion render path: renderChildRows had zero consumers and is removed; renderSubComponent (colspan cell) is the single path - the four consumers passing dead no-op renderSubComponent and getRowCanExpand boilerplate drop it - loading and empty defaults become generic (Loading... / No results) instead of log-specific The characterization tests from the previous commit pass unchanged except the dead child-rows path test, replaced by a reorder-stability test for injected getRowId plus coverage of the new generic defaults. First tremor removal of the tables track; view_logs/table.tsx no longer imports @tremor/react. * test(ui): assert child rows hidden before expansion in DataTable test * fix(ui): suppress row hover on DataTable placeholder rows * feat(ui): polish DataTable with skeleton loading, header band, and numeric column alignment * feat(ui): shape DataTable skeletons per column and keep stale rows during refetch * revert(ui): drop DataTable skeleton loading, restore text loading row * fix(ui): clip DataTable to its rounded wrapper and right-align Duration/TTFT values --- ui/litellm-dashboard/eslint-metrics.json | 2 +- ui/litellm-dashboard/eslint-suppressions.json | 5 -- .../components/EntityUsage/TopKeyView.tsx | 9 +-- .../components/EntityUsage/TopModelView.tsx | 12 ++- .../components/mcp_tools/MCPToolsetsTab.tsx | 2 - .../src/components/pass_through_settings.tsx | 2 - .../src/components/view_logs/columns.tsx | 10 ++- .../src/components/view_logs/index.tsx | 1 + .../src/components/view_logs/table.test.tsx | 81 ++++++++++++++++--- .../src/components/view_logs/table.tsx | 78 +++++++++--------- 10 files changed, 124 insertions(+), 78 deletions(-) diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 219cb0580e7..51cef1169f9 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,5 +1,5 @@ { - "@typescript-eslint/no-explicit-any": 1990, + "@typescript-eslint/no-explicit-any": 1988, "complexity": 128, "max-depth": 59, "no-console": 15 diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 1c8f92b720f..67b19471aaf 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2077,11 +2077,6 @@ "count": 1 } }, - "src/components/view_logs/table.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_user_spend.tsx": { "react-hooks/set-state-in-effect": { "count": 2 diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx index d0748583c30..40bc41b3e8c 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx @@ -164,6 +164,7 @@ const TopKeyView: React.FC = ({ topKeys, teams, showTags = fals const spendColumn = { header: "Spend (USD)", accessorKey: "spend", + meta: { numeric: true }, cell: (info: any) => { const value = info.getValue(); return value > 0 && value < 0.01 ? "<$0.01" : `$${formatNumberWithCommas(value, 2)}`; @@ -247,13 +248,7 @@ const TopKeyView: React.FC = ({ topKeys, teams, showTags = fals
) : (
- <>} - getRowCanExpand={() => false} - isLoading={false} - /> +
)} diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx index c69ba42f182..7562ef06a03 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx @@ -30,6 +30,7 @@ export default function TopModelView({ topModels, topModelsLimit, setTopModelsLi { header: "Spend (USD)", accessorKey: "spend", + meta: { numeric: true }, cell: (info: any) => { const value = info.getValue(); return `$${formatNumberWithCommas(value, 2)}`; @@ -38,16 +39,19 @@ export default function TopModelView({ topModels, topModelsLimit, setTopModelsLi { header: "Successful", accessorKey: "successful_requests", + meta: { numeric: true }, cell: (info: any) => {info.getValue()?.toLocaleString() || 0}, }, { header: "Failed", accessorKey: "failed_requests", + meta: { numeric: true }, cell: (info: any) => {info.getValue()?.toLocaleString() || 0}, }, { header: "Tokens", accessorKey: "tokens", + meta: { numeric: true }, cell: (info: any) => info.getValue()?.toLocaleString() || 0, }, ]; @@ -99,13 +103,7 @@ export default function TopModelView({ topModels, topModelsLimit, setTopModelsLi
) : (
- <>} - getRowCanExpand={() => false} - isLoading={false} - /> +
)} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx index 0198b830229..546df9ebc4b 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx @@ -509,8 +509,6 @@ export function MCPToolsetsTab({ accessToken, userRole }: MCPToolsetsTabProps) {
} - getRowCanExpand={() => false} isLoading={isLoading} noDataMessage="No toolsets yet. Click 'New Toolset' to create one." loadingMessage="Loading toolsets..." diff --git a/ui/litellm-dashboard/src/components/pass_through_settings.tsx b/ui/litellm-dashboard/src/components/pass_through_settings.tsx index 0fdd9c632bf..63fe0f92961 100644 --- a/ui/litellm-dashboard/src/components/pass_through_settings.tsx +++ b/ui/litellm-dashboard/src/components/pass_through_settings.tsx @@ -263,8 +263,6 @@ const PassThroughSettings: React.FC = ({
} - getRowCanExpand={() => false} isLoading={false} noDataMessage="No pass-through endpoints configured" /> diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index 0508c562df8..7452992ed59 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -231,13 +231,14 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] : "Cost", accessorKey: "spend", size: 110, + meta: { numeric: true }, cell: (info: any) => { const row = info.row.original; const mcpCount = row.mcp_tool_call_count || 0; const mcpSpend = row.mcp_tool_call_spend || 0; return ( -
+
{getSpendString(info.getValue() || 0)} @@ -263,13 +264,14 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] ) : "Duration (s)", accessorKey: "request_duration_ms", + meta: { numeric: true }, cell: (info: any) => { const ms = info.getValue(); if (ms == null) return -; const seconds = (ms / 1000).toFixed(2); return ( - {seconds} + {seconds} ); }, @@ -287,6 +289,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] ) : "TTFT (s)", accessorKey: "completionStartTime", + meta: { numeric: true }, cell: (info: any) => { const row = info.row.original; const completionStartTime = info.getValue(); @@ -298,7 +301,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] const ttftSeconds = (ttftMs / 1000).toFixed(2); return ( - {ttftSeconds} + {ttftSeconds} ); }, @@ -395,6 +398,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] : "Tokens", accessorKey: "total_tokens", size: 140, + meta: { numeric: true }, cell: (info: any) => { const row = info.row.original; return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index 265e331e6a9..ee08712e56b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -287,6 +287,7 @@ export default function SpendLogsTable({ accessToken, token, userRole, userID, p row.request_id} onRowClick={handleRowClick} isLoading={isLogsLoading} /> diff --git a/ui/litellm-dashboard/src/components/view_logs/table.test.tsx b/ui/litellm-dashboard/src/components/view_logs/table.test.tsx index 7299d280769..9a8469cfeba 100644 --- a/ui/litellm-dashboard/src/components/view_logs/table.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/table.test.tsx @@ -74,6 +74,35 @@ describe("DataTable states", () => { expect(screen.getByText("Nothing here")).toBeInTheDocument(); }); + it("falls back to generic loading and empty defaults", () => { + const { rerender } = render(); + expect(screen.getByText("Loading...")).toBeInTheDocument(); + + rerender(); + expect(screen.getByText("No results")).toBeInTheDocument(); + }); + + it("suppresses the primitive's row hover on loading, empty, and expansion placeholder rows", async () => { + const user = userEvent.setup(); + const { rerender } = render(); + expect(screen.getByText("Loading...").closest("tr")).toHaveClass("hover:bg-transparent"); + + rerender(); + expect(screen.getByText("No results").closest("tr")).toHaveClass("hover:bg-transparent"); + + rerender( + true} + renderSubComponent={({ row }) =>
details for {row.original.request_id}
} + />, + ); + await user.click(screen.getByRole("button", { name: "expand r1" })); + expect(screen.getByText("details for r1").closest("tr")).toHaveClass("hover:bg-transparent"); + expect(screen.getByText("alpha").closest("tr")).not.toHaveClass("hover:bg-transparent"); + }); + it("renders row data through plain TanStack column defs, including custom cell renderers", () => { const columns: ColumnDef[] = [ { header: "A", accessorKey: "a" }, @@ -84,6 +113,29 @@ describe("DataTable states", () => { expect(screen.getByText("alpha")).toBeInTheDocument(); expect(screen.getByText("custom:beta")).toBeInTheDocument(); }); + + it("clips the table to the rounded wrapper so the header band cannot bleed past the corners", () => { + const { container } = render(); + + const wrapper = container.firstElementChild; + expect(wrapper).toHaveClass("rounded-lg", "overflow-hidden"); + }); + + it("right-aligns headers and cells with tabular figures for numeric meta columns", () => { + const columns: ColumnDef[] = [ + { header: "A", accessorKey: "a" }, + { header: "B", accessorKey: "b", meta: { numeric: true } }, + ]; + render(); + + const headers = screen.getAllByRole("columnheader"); + expect(headers[1].querySelector("div")).toHaveClass("justify-end"); + expect(headers[0].querySelector("div")).not.toHaveClass("justify-end"); + + const cells = screen.getAllByRole("cell"); + expect(cells[1]).toHaveClass("text-right", "tabular-nums"); + expect(cells[0]).not.toHaveClass("text-right"); + }); }); describe("DataTable row interaction", () => { @@ -129,28 +181,33 @@ describe("DataTable expansion", () => { expect(screen.queryByText("details for r1")).not.toBeInTheDocument(); }); - it("renders child rows as sibling table rows (child-rows path)", async () => { + it("keeps expansion attached to the same row through data reorders when getRowId is injected", async () => { const user = userEvent.setup(); - render( + const { rerender } = render( row.request_id} getRowCanExpand={() => true} - renderChildRows={({ row }) => ( - - child of {row.original.request_id} - - )} + renderSubComponent={({ row }) =>
details for {row.original.request_id}
} />, ); - expect(screen.queryByText("child of r2")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "expand r1" })); + expect(screen.getByText("details for r1")).toBeInTheDocument(); - await user.click(screen.getByRole("button", { name: "expand r2" })); + rerender( + row.request_id} + getRowCanExpand={() => true} + renderSubComponent={({ row }) =>
details for {row.original.request_id}
} + />, + ); - const childCell = screen.getByText("child of r2"); - expect(childCell.closest("tr")).not.toBeNull(); - expect(within(screen.getByRole("table")).getByText("child of r2")).toBeInTheDocument(); + expect(screen.getByText("details for r1")).toBeInTheDocument(); + expect(screen.queryByText("details for r2")).not.toBeInTheDocument(); }); it("does not expand rows when getRowCanExpand is missing even if a renderer is provided", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/table.tsx b/ui/litellm-dashboard/src/components/view_logs/table.tsx index 4510cc9a1f0..c96f34f9b93 100644 --- a/ui/litellm-dashboard/src/components/view_logs/table.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/table.tsx @@ -1,6 +1,7 @@ import { Fragment, useState } from "react"; import { ColumnDef, + RowData, flexRender, getCoreRowModel, getExpandedRowModel, @@ -10,16 +11,21 @@ import { SortingState, } from "@tanstack/react-table"; -import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; +import { Table, TableHeader, TableHead, TableBody, TableRow, TableCell } from "@/components/ui/table"; + +declare module "@tanstack/react-table" { + interface ColumnMeta { + numeric?: boolean; + } +} interface DataTableProps { data: TData[]; columns: ColumnDef[]; + getRowId?: (row: TData, index: number) => string; onRowClick?: (row: TData) => void; - /** Renders inside a single colspan cell (used by audit logs) */ + /** Renders inside a single colspan cell */ renderSubComponent?: (props: { row: Row }) => React.ReactElement; - /** Renders directly in tbody as sibling table rows (used by MCP children) */ - renderChildRows?: (props: { row: Row }) => React.ReactNode; getRowCanExpand?: (row: Row) => boolean; isLoading?: boolean; loadingMessage?: string; @@ -31,16 +37,16 @@ interface DataTableProps { export function DataTable({ data = [], columns, + getRowId, onRowClick, renderSubComponent, - renderChildRows, getRowCanExpand, isLoading = false, - loadingMessage = "🚅 Loading logs...", - noDataMessage = "No logs found", + loadingMessage = "Loading...", + noDataMessage = "No results", enableSorting = false, }: DataTableProps) { - const supportsExpansion = !!(renderSubComponent || renderChildRows) && !!getRowCanExpand; + const supportsExpansion = !!renderSubComponent && !!getRowCanExpand; const hasExplicitColumnSizes = columns.some((column) => column.size !== undefined); const [sorting, setSorting] = useState([]); @@ -55,58 +61,56 @@ export function DataTable({ enableSortingRemoval: false, }), ...(supportsExpansion && { getRowCanExpand }), - getRowId: (row: TData, index: number) => { - const _row: any = row as any; - return _row?.request_id ?? String(index); - }, + ...(getRowId && { getRowId }), getCoreRowModel: getCoreRowModel(), ...(enableSorting && { getSortedRowModel: getSortedRowModel() }), ...(supportsExpansion && { getExpandedRowModel: getExpandedRowModel() }), }); - const tableClassName = hasExplicitColumnSizes - ? "[&_td]:py-0.5 [&_th]:py-1 [&_table]:table-fixed" - : "[&_td]:py-0.5 [&_th]:py-1 table-fixed w-full box-border"; + const tableClassName = hasExplicitColumnSizes ? "table-fixed" : "table-fixed w-full box-border"; const tableStyle = hasExplicitColumnSizes ? { minWidth: table.getCenterTotalSize() } : { minWidth: "400px" }; return ( -
+
- + {table.getHeaderGroups().map((headerGroup) => ( - + {headerGroup.headers.map((header) => { const canSort = enableSorting && header.column.getCanSort(); const isSorted = header.column.getIsSorted(); + const numeric = header.column.columnDef.meta?.numeric; return ( - {header.isPlaceholder ? null : ( -
+
{flexRender(header.column.columnDef.header, header.getContext())} {canSort && ( - + {isSorted === "asc" ? "↑" : isSorted === "desc" ? "↓" : "⇅"} )}
)} - + ); })} ))} - + {isLoading ? ( - + -
+

{loadingMessage}

@@ -115,13 +119,15 @@ export function DataTable({ table.getRowModel().rows.map((row) => ( onRowClick?.(row.original)} > {row.getVisibleCells().map((cell) => ( {flexRender(cell.column.columnDef.cell, cell.getContext())} @@ -129,12 +135,8 @@ export function DataTable({ ))} - {/* Child rows rendered as real table rows (MCP children) */} - {supportsExpansion && row.getIsExpanded() && renderChildRows && renderChildRows({ row })} - - {/* Legacy sub-component in colspan cell (audit logs) */} - {supportsExpansion && row.getIsExpanded() && renderSubComponent && !renderChildRows && ( - + {supportsExpansion && row.getIsExpanded() && renderSubComponent && ( +
{renderSubComponent({ row })}
 @@ -143,11 +145,9 @@ export function DataTable({ )) ) : ( - - -
-

{noDataMessage}

-
+ + +

{noDataMessage}

 )} From 0f1e29b33486ba6e1600fb93de7214e57e54047d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:00:24 -0700 Subject: [PATCH 021/101] fix(bedrock): preserve cache_control ttl on message-level cache points (#32538) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../prompt_templates/factory.py | 12 ++- ...llm_core_utils_prompt_templates_factory.py | 81 +++++++++++++++++++ 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index c1635158d3b..06abb591717 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4377,6 +4377,7 @@ class BedrockConverseMessagesProcessor: _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast(OpenAIMessageContentListBlock, element), block_type="content_block", + model=model, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -4384,7 +4385,7 @@ class BedrockConverseMessagesProcessor: elif message_block["content"] and isinstance(message_block["content"], str): _part = BedrockContentBlock(text=messages[msg_i]["content"]) _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( - message_block, block_type="content_block" + message_block, block_type="content_block", model=model ) user_content.append(_part) if _cache_point_block is not None: @@ -4509,6 +4510,7 @@ class BedrockConverseMessagesProcessor: _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast(OpenAIMessageContentListBlock, element), block_type="content_block", + model=model, ) if _cache_point_block is not None: assistants_parts.append(_cache_point_block) @@ -4520,7 +4522,7 @@ class BedrockConverseMessagesProcessor: # If content is empty/whitespace, skip it (don't add a placeholder) # Add cache point block for assistant string content _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( - assistant_message_block, block_type="content_block" + assistant_message_block, block_type="content_block", model=model ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) @@ -4745,6 +4747,7 @@ def _bedrock_converse_messages_pt( _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast(OpenAIMessageContentListBlock, element), block_type="content_block", + model=model, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -4752,7 +4755,7 @@ def _bedrock_converse_messages_pt( elif message_block["content"] and isinstance(message_block["content"], str): _part = BedrockContentBlock(text=messages[msg_i]["content"]) _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( - message_block, block_type="content_block" + message_block, block_type="content_block", model=model ) user_content.append(_part) if _cache_point_block is not None: @@ -4882,6 +4885,7 @@ def _bedrock_converse_messages_pt( _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast(OpenAIMessageContentListBlock, element), block_type="content_block", + model=model, ) if _cache_point_block is not None: assistants_parts.append(_cache_point_block) @@ -4892,7 +4896,7 @@ def _bedrock_converse_messages_pt( assistant_content.append(BedrockContentBlock(text=_assistant_content)) # Add cache point block for assistant string content _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( - assistant_message_block, block_type="content_block" + assistant_message_block, block_type="content_block", model=model ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 1d5289737f1..bcda88ea609 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -3085,3 +3085,84 @@ def test_bedrock_converse_messages_pt_document_rejects_url_source(): _bedrock_converse_messages_pt( messages, "anthropic.claude-sonnet-4-6", "bedrock" ) + + +def _collect_cache_points(blocks): + return [ + block["cachePoint"] + for message in blocks + for block in message["content"] + if "cachePoint" in block + ] + + +@pytest.mark.parametrize( + "messages", + [ + [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "conversation history", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + }, + ], + [ + {"role": "user", "content": "hello"}, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "assistant reply", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + }, + ], + ], +) +def test_bedrock_converse_message_level_cache_point_preserves_ttl(messages): + """ + Regression for https://github.com/BerriAI/litellm/issues/32154: message-level + cache_control ttl was silently dropped because the message-level + _get_cache_point_block call sites never passed model=, so multi-turn prefixes + fell back to the 5m default while the system prompt kept 1h, churning the + cache every turn on models like Opus 4.8. + """ + result = _bedrock_converse_messages_pt( + messages=messages, + model="eu.anthropic.claude-opus-4-8", + llm_provider="bedrock", + ) + + cache_points = _collect_cache_points(result) + assert cache_points == [{"type": "default", "ttl": "1h"}] + + +@pytest.mark.asyncio +async def test_bedrock_converse_message_level_cache_point_preserves_ttl_async(): + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "conversation history", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ], + }, + ] + + result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="eu.anthropic.claude-opus-4-8", + llm_provider="bedrock", + ) + + assert _collect_cache_points(result) == [{"type": "default", "ttl": "1h"}] From 7b2742777d31d2c7af6eeb5e1d3a3770d3570c81 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Jul 2026 14:07:29 -0700 Subject: [PATCH 022/101] refactor(ui): dedupe /health/license fetch via shared useLicenseInfo hook UsageIndicator was fetching /health/license through its own useEffect while the new expiry banner fetches the same endpoint via useLicenseInfo, so an admin with the usage widget open made two identical calls per page load. Point UsageIndicator at useLicenseInfo too; both callers now share one React Query cache entry, collapsing it back to a single request. The null/error semantics are preserved (data ?? null matches the previous catch-to-null), and license errors never fed the widget's error state before either --- .../src/components/UsageIndicator.test.tsx | 30 +++++++++++-------- .../src/components/UsageIndicator.tsx | 12 ++++---- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/ui/litellm-dashboard/src/components/UsageIndicator.test.tsx b/ui/litellm-dashboard/src/components/UsageIndicator.test.tsx index 8c7c15bc5a5..ad27fbcd74f 100644 --- a/ui/litellm-dashboard/src/components/UsageIndicator.test.tsx +++ b/ui/litellm-dashboard/src/components/UsageIndicator.test.tsx @@ -2,6 +2,7 @@ import React from "react"; import { describe, it, expect, vi, beforeEach } from "vitest"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import UsageIndicator from "./UsageIndicator"; vi.mock("./networking", () => ({ @@ -17,6 +18,11 @@ import { getRemainingUsers } from "./networking"; const mockGetRemainingUsers = vi.mocked(getRemainingUsers); +const renderWithClient = (ui: React.ReactElement) => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render({ui}); +}; + const DEFAULT_USAGE_DATA = { total_users: 100, total_users_used: 1, @@ -33,7 +39,7 @@ describe("UsageIndicator", () => { }); it("should render when given access token and usage data loads", async () => { - render(); + renderWithClient(); await screen.findByText("Usage"); @@ -41,7 +47,7 @@ describe("UsageIndicator", () => { }); it("should not show Near limit when users usage is below 80% (1/100 -> 1%)", async () => { - render(); + renderWithClient(); await screen.findByText("Usage"); @@ -58,7 +64,7 @@ describe("UsageIndicator", () => { total_users_remaining: null, }); - render(); + renderWithClient(); await waitFor(() => { expect(screen.queryByText("Usage")).not.toBeInTheDocument(); @@ -76,7 +82,7 @@ describe("UsageIndicator", () => { total_teams_remaining: 1, }); - render(); + renderWithClient(); await screen.findByText("Usage"); @@ -94,7 +100,7 @@ describe("UsageIndicator", () => { total_teams_remaining: null, }); - render(); + renderWithClient(); await screen.findByText("Usage"); @@ -112,7 +118,7 @@ describe("UsageIndicator", () => { total_teams_remaining: -2, }); - render(); + renderWithClient(); await screen.findByText("Usage"); @@ -121,7 +127,7 @@ describe("UsageIndicator", () => { }); it("should render nothing when accessToken is null", () => { - render(); + renderWithClient(); expect(mockGetRemainingUsers).not.toHaveBeenCalled(); expect(screen.queryByText("Usage")).not.toBeInTheDocument(); @@ -131,7 +137,7 @@ describe("UsageIndicator", () => { const { useDisableUsageIndicator } = await import("@/app/(dashboard)/hooks/useDisableUsageIndicator"); (useDisableUsageIndicator as ReturnType).mockReturnValue(true); - render(); + renderWithClient(); await waitFor(() => { expect(screen.queryByText("Usage")).not.toBeInTheDocument(); @@ -143,7 +149,7 @@ describe("UsageIndicator", () => { it("should show Loading while fetching", () => { mockGetRemainingUsers.mockImplementation(() => new Promise(() => {})); - render(); + renderWithClient(); expect(screen.getByText("Loading...")).toBeInTheDocument(); }); @@ -152,7 +158,7 @@ describe("UsageIndicator", () => { const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); mockGetRemainingUsers.mockRejectedValue(new Error("Network error")); - render(); + renderWithClient(); expect(await screen.findByText("Failed to load usage data")).toBeInTheDocument(); @@ -161,7 +167,7 @@ describe("UsageIndicator", () => { it("should minimize when user clicks minimize button", async () => { const user = userEvent.setup(); - render(); + renderWithClient(); await screen.findByText("Usage"); @@ -174,7 +180,7 @@ describe("UsageIndicator", () => { it("should restore from minimized when user clicks restore button", async () => { const user = userEvent.setup(); - render(); + renderWithClient(); await screen.findByText("Usage"); diff --git a/ui/litellm-dashboard/src/components/UsageIndicator.tsx b/ui/litellm-dashboard/src/components/UsageIndicator.tsx index 8165f8eb6f6..6e7b5e9ec60 100644 --- a/ui/litellm-dashboard/src/components/UsageIndicator.tsx +++ b/ui/litellm-dashboard/src/components/UsageIndicator.tsx @@ -12,10 +12,11 @@ import { Users, } from "lucide-react"; import { useEffect, useState } from "react"; -import { getRemainingUsers, getLicenseInfo, LicenseInfo } from "./networking"; +import { getRemainingUsers } from "./networking"; import { cn } from "@/lib/cva.config"; import { getDaysUntilExpiration } from "@/utils/licenseUtils"; +import { useLicenseInfo } from "@/app/(dashboard)/hooks/license/useLicenseInfo"; interface UsageIndicatorProps { accessToken: string | null; @@ -48,10 +49,11 @@ export default function UsageIndicator({ accessToken, width = 220 }: UsageIndica const [isExpanded, setIsExpanded] = useState(false); const [isMinimized, setIsMinimized] = useState(false); const [data, setData] = useState(null); - const [licenseInfo, setLicenseInfo] = useState(null); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(null); + const licenseInfo = useLicenseInfo(accessToken).data ?? null; + useEffect(() => { const fetchData = async () => { if (!accessToken) return; @@ -60,12 +62,8 @@ export default function UsageIndicator({ accessToken, width = 220 }: UsageIndica setError(null); try { - const [usageResult, licenseResult] = await Promise.all([ - getRemainingUsers(accessToken), - getLicenseInfo(accessToken).catch(() => null), // Don't fail if license endpoint unavailable - ]); + const usageResult = await getRemainingUsers(accessToken); setData(usageResult); - setLicenseInfo(licenseResult); } catch (err) { console.error("Failed to fetch usage data:", err); setError("Failed to load usage data"); From 5973d9fd2b0d074f963e95fdae2b9c1aef3d88bc Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 8 Jul 2026 14:32:16 -0700 Subject: [PATCH 023/101] feat(ui): add eslint rules for nested ternaries, large inline object args, and long condition chains (#32415) * feat(ui): add eslint rules for nested ternaries, large inline object args, and long condition chains Adds three dashboard lint rules to keep new code readable. Nested ternaries are banned outright via the built-in no-nested-ternary, with the 265 existing occurrences grandfathered in eslint-suppressions.json so only new ones fail. Two custom rules ship as a small local plugin under scripts/eslint-rules: no-large-inline-object-arg flags object literals with 4+ properties passed straight into a call, nudging toward a named variable, and no-long-condition-chain flags boolean expressions that combine 4+ conditions, nudging toward a named boolean. Both are warnings tracked on the existing budget ratchet (eslint-budgets.json + eslint-metrics.json) with headroom above the current counts, so they ratchet down over time rather than freezing a baseline. Both thresholds are configurable rule options and covered by RuleTester unit tests. * fix(ui): scope no-long-condition-chain to boolean operators, not nullish Greptile flagged that the rule counted nullish-coalescing chains the same as &&/|| chains, so a 4-part `a ?? b ?? c ?? d` fallback surfaced "Boolean expression combines 4 conditions", which is inaccurate since a `??` fallback is value defaulting, not a condition. Restrict the visitor to && / || nodes so `??` chains are treated as leaves, while a boolean chain nested inside a `??` is still caught. Drops 6 miscounted occurrences (240 -> 234). * chore(ui): sync lint metrics and suppressions with staging Merge advanced the base branch, adding one no-large-inline-object-arg occurrence (508 -> 509) and making one grandfathered react-hooks suppression stale. Regenerate eslint-metrics.json and prune the suppression so the budget/drift gate passes. * chore(ui): sync lint metrics with staging Merge advanced the base, adding four no-large-inline-object-arg occurrences (509 -> 513). Regenerate eslint-metrics.json so the drift gate passes. --- ui/litellm-dashboard/eslint-budgets.json | 4 +- ui/litellm-dashboard/eslint-metrics.json | 2 + ui/litellm-dashboard/eslint-suppressions.json | 442 +++++++++++++++++- ui/litellm-dashboard/eslint.config.mjs | 6 +- .../scripts/eslint-rules/index.mjs | 11 + .../no-large-inline-object-arg.mjs | 41 ++ .../eslint-rules/no-long-condition-chain.mjs | 41 ++ .../no-large-inline-object-arg.test.ts | 46 ++ .../no-long-condition-chain.test.ts | 51 ++ 9 files changed, 641 insertions(+), 3 deletions(-) create mode 100644 ui/litellm-dashboard/scripts/eslint-rules/index.mjs create mode 100644 ui/litellm-dashboard/scripts/eslint-rules/no-large-inline-object-arg.mjs create mode 100644 ui/litellm-dashboard/scripts/eslint-rules/no-long-condition-chain.mjs create mode 100644 ui/litellm-dashboard/tests/eslint-rules/no-large-inline-object-arg.test.ts create mode 100644 ui/litellm-dashboard/tests/eslint-rules/no-long-condition-chain.test.ts diff --git a/ui/litellm-dashboard/eslint-budgets.json b/ui/litellm-dashboard/eslint-budgets.json index 8dedb9ac9ca..f08e1bb6160 100644 --- a/ui/litellm-dashboard/eslint-budgets.json +++ b/ui/litellm-dashboard/eslint-budgets.json @@ -2,5 +2,7 @@ "@typescript-eslint/no-explicit-any": { "max": 2040, "target": 1500 }, "no-console": { "max": 484, "target": 0 }, "complexity": { "max": 140, "target": 80 }, - "max-depth": { "max": 70, "target": 30 } + "max-depth": { "max": 70, "target": 30 }, + "local/no-large-inline-object-arg": { "max": 560, "target": 300 }, + "local/no-long-condition-chain": { "max": 265, "target": 120 } } diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 51cef1169f9..f4dc89c5b80 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,6 +1,8 @@ { "@typescript-eslint/no-explicit-any": 1988, "complexity": 128, + "local/no-large-inline-object-arg": 513, + "local/no-long-condition-chain": 233, "max-depth": 59, "no-console": 15 } diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 67b19471aaf..b077338c75b 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1,4 +1,9 @@ { + "scripts/check-lint-budgets.mjs": { + "no-nested-ternary": { + "count": 1 + } + }, "src/app/(dashboard)/api-reference/APIReferenceView.tsx": { "no-restricted-imports": { "count": 1 @@ -64,6 +69,9 @@ } }, "src/app/(dashboard)/cost-tracking/components/cost_tracking_settings.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } @@ -133,11 +141,21 @@ "count": 1 } }, + "src/app/(dashboard)/guardrails-monitor/components/GuardrailDetail.tsx": { + "no-nested-ternary": { + "count": 3 + } + }, "src/app/(dashboard)/guardrails-monitor/components/GuardrailsMonitorView.tsx": { "no-restricted-imports": { "count": 1 } }, + "src/app/(dashboard)/guardrails-monitor/components/GuardrailsOverview.tsx": { + "no-nested-ternary": { + "count": 8 + } + }, "src/app/(dashboard)/guardrails-monitor/components/ScoreChart.test.tsx": { "react/display-name": { "count": 1 @@ -343,6 +361,9 @@ } }, "src/app/(dashboard)/models-and-endpoints/components/ModelRetrySettingsTab.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -361,6 +382,9 @@ } }, "src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx": { + "no-nested-ternary": { + "count": 2 + }, "react-hooks/set-state-in-effect": { "count": 5 } @@ -370,7 +394,15 @@ "count": 1 } }, + "src/app/(dashboard)/playground/components/chat_ui/ChatMessageBubble.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx": { + "no-nested-ternary": { + "count": 7 + }, "no-restricted-imports": { "count": 1 }, @@ -382,6 +414,9 @@ } }, "src/app/(dashboard)/playground/components/chat_ui/CodeInterpreterOutput.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-syntax": { "count": 2 } @@ -392,6 +427,9 @@ } }, "src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx": { + "no-nested-ternary": { + "count": 2 + }, "react-hooks/immutability": { "count": 2 }, @@ -400,16 +438,27 @@ } }, "src/app/(dashboard)/playground/components/compareUI/CompareUI.tsx": { + "no-nested-ternary": { + "count": 4 + }, "react-hooks/set-state-in-effect": { "count": 1 } }, + "src/app/(dashboard)/playground/components/compareUI/components/MessageDisplay.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/app/(dashboard)/playground/components/compareUI/components/ModelSelector.tsx": { "no-restricted-imports": { "count": 1 } }, "src/app/(dashboard)/playground/components/complianceUI/ComplianceUI.tsx": { + "no-nested-ternary": { + "count": 8 + }, "react-hooks/preserve-manual-memoization": { "count": 3 } @@ -474,6 +523,9 @@ } }, "src/app/(dashboard)/projects/components/ProjectDetailsPage.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 } @@ -499,6 +551,9 @@ } }, "src/app/(dashboard)/prompts/components/index.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -517,6 +572,9 @@ } }, "src/app/(dashboard)/prompts/components/prompt_editor_view/PromptCodeSnippets.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -550,6 +608,9 @@ } }, "src/app/(dashboard)/prompts/components/prompt_editor_view/VersionHistorySidePanel.tsx": { + "no-nested-ternary": { + "count": 1 + }, "react-hooks/immutability": { "count": 1 } @@ -570,6 +631,9 @@ } }, "src/app/(dashboard)/prompts/components/prompt_info.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 }, @@ -578,6 +642,9 @@ } }, "src/app/(dashboard)/prompts/components/prompt_table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -635,6 +702,9 @@ } }, "src/app/(dashboard)/users/_components/user_edit_view.test.tsx": { + "no-nested-ternary": { + "count": 1 + }, "react/display-name": { "count": 1 } @@ -648,6 +718,9 @@ } }, "src/app/(dashboard)/users/_components/view_users.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -664,6 +737,9 @@ } }, "src/app/(dashboard)/users/_components/view_users/table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -677,6 +753,9 @@ } }, "src/app/(dashboard)/workflows/WorkflowRuns.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-syntax": { "count": 3 }, @@ -684,6 +763,11 @@ "count": 1 } }, + "src/app/chat/page.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/app/login/LoginPage.tsx": { "react-hooks/set-state-in-effect": { "count": 2 @@ -715,6 +799,9 @@ } }, "src/components/AIHub/ModelHubTable.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -746,6 +833,9 @@ } }, "src/components/AIHub/forms/MakeMCPPublicForm.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 }, @@ -778,11 +868,17 @@ } }, "src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } }, "src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -815,11 +911,26 @@ "count": 1 } }, + "src/components/GuardrailSettingsView.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, + "src/components/GuardrailsMonitor/LogViewer.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/HelpLink.test.tsx": { "unused-imports/no-unused-imports": { "count": 1 } }, + "src/components/ModelSelect/PaginatedModelSelect/PaginatedModelSelect.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/Navbar/BlogDropdown/BlogDropdown.test.tsx": { "max-nested-callbacks": { "count": 12 @@ -836,6 +947,9 @@ } }, "src/components/OldTeams.tsx": { + "no-nested-ternary": { + "count": 4 + }, "no-restricted-imports": { "count": 1 }, @@ -856,7 +970,15 @@ "count": 1 } }, + "src/components/Settings/AdminSettings/HashicorpVault/HashicorpVault.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx": { + "no-nested-ternary": { + "count": 1 + }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -871,6 +993,11 @@ "count": 1 } }, + "src/components/Settings/AdminSettings/SSOSettings/RedactableField.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.test.tsx": { "max-nested-callbacks": { "count": 4 @@ -881,7 +1008,15 @@ "count": 2 } }, + "src/components/Settings/AdminSettings/UISettings/UISettings.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -907,12 +1042,20 @@ "count": 1 } }, + "src/components/TeamSSOSettings.test.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/ToolDetail.tsx": { "unused-imports/no-unused-imports": { "count": 2 } }, "src/components/ToolPolicies.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -932,6 +1075,9 @@ } }, "src/components/UsageIndicator.tsx": { + "no-nested-ternary": { + "count": 4 + }, "no-restricted-imports": { "count": 1 }, @@ -949,6 +1095,11 @@ "count": 1 } }, + "src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/UsagePage/components/EntityUsage/EntityUsage.tsx": { "no-restricted-imports": { "count": 1 @@ -975,11 +1126,17 @@ } }, "src/components/UsagePage/components/UsageAIChatPanel.tsx": { + "no-nested-ternary": { + "count": 1 + }, "react-hooks/immutability": { "count": 1 } }, "src/components/UsagePage/components/UsagePageView.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -999,16 +1156,25 @@ } }, "src/components/VirtualKeysPage/VirtualKeysTable.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } }, "src/components/activity_metrics.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } }, "src/components/add_model/AddModelForm.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1045,11 +1211,22 @@ } }, "src/components/add_model/litellm_model_name.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } }, + "src/components/add_model/model_connection_test.tsx": { + "no-nested-ternary": { + "count": 2 + } + }, "src/components/add_model/provider_specific_fields.tsx": { + "no-nested-ternary": { + "count": 5 + }, "no-restricted-imports": { "count": 1 }, @@ -1079,6 +1256,9 @@ } }, "src/components/agents/add_agent_form.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 }, @@ -1102,7 +1282,15 @@ "count": 1 } }, + "src/components/agents/agent_form_fields.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/agents/agent_info.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -1110,7 +1298,20 @@ "count": 1 } }, + "src/components/agents/agent_virtual_keys.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, + "src/components/agents/dynamic_agent_form_fields.tsx": { + "no-nested-ternary": { + "count": 2 + } + }, "src/components/alerting/dynamic_form.tsx": { + "no-nested-ternary": { + "count": 4 + }, "no-restricted-imports": { "count": 1 } @@ -1123,6 +1324,31 @@ "count": 1 } }, + "src/components/chat/KeysPanel.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, + "src/components/chat/MCPAppsPanel.tsx": { + "no-nested-ternary": { + "count": 7 + } + }, + "src/components/chat/MCPConnectPicker.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, + "src/components/chat/MCPCredentialsTab.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, + "src/components/chat/UsagePanel.tsx": { + "no-nested-ternary": { + "count": 2 + } + }, "src/components/claude_code_plugins.tsx": { "no-restricted-imports": { "count": 1 @@ -1145,6 +1371,9 @@ } }, "src/components/claude_code_plugins/plugin_table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1235,6 +1464,9 @@ } }, "src/components/common_components/chartUtils.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1250,6 +1482,9 @@ } }, "src/components/common_components/simple_table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1286,6 +1521,9 @@ } }, "src/components/general_settings.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 2 } @@ -1300,17 +1538,28 @@ "count": 1 } }, + "src/components/guardrails/GuardrailTestPlayground.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/guardrails/GuardrailTestResults.tsx": { "no-restricted-imports": { "count": 1 } }, "src/components/guardrails/TeamGuardrailsTab.tsx": { + "no-nested-ternary": { + "count": 2 + }, "react-hooks/set-state-in-effect": { "count": 1 } }, "src/components/guardrails/add_guardrail_form.tsx": { + "no-nested-ternary": { + "count": 4 + }, "react-hooks/set-state-in-effect": { "count": 1 }, @@ -1319,11 +1568,17 @@ } }, "src/components/guardrails/content_filter/CompetitorIntentConfiguration.tsx": { + "no-nested-ternary": { + "count": 1 + }, "react-hooks/set-state-in-effect": { "count": 1 } }, "src/components/guardrails/content_filter/ContentCategoryConfiguration.tsx": { + "no-nested-ternary": { + "count": 3 + }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1342,6 +1597,9 @@ } }, "src/components/guardrails/custom_code/CustomCodeModal.tsx": { + "no-nested-ternary": { + "count": 6 + }, "no-restricted-imports": { "count": 1 }, @@ -1372,16 +1630,25 @@ } }, "src/components/guardrails/guardrail_optional_params.tsx": { + "no-nested-ternary": { + "count": 5 + }, "react-hooks/set-state-in-effect": { "count": 1 } }, "src/components/guardrails/guardrail_provider_fields.tsx": { + "no-nested-ternary": { + "count": 5 + }, "react-hooks/set-state-in-effect": { "count": 1 } }, "src/components/guardrails/guardrail_table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 2 } @@ -1407,6 +1674,9 @@ "src/components/llm_calls/chat_completion.tsx": { "max-params": { "count": 1 + }, + "no-nested-ternary": { + "count": 1 } }, "src/components/llm_calls/responses_api.tsx": { @@ -1444,7 +1714,15 @@ "count": 1 } }, + "src/components/mcp_tools/MCPToolArgumentsForm.tsx": { + "no-nested-ternary": { + "count": 5 + } + }, "src/components/mcp_tools/MCPToolsetsTab.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -1456,11 +1734,17 @@ } }, "src/components/mcp_tools/McpCrudPermissionPanel.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 } }, "src/components/mcp_tools/OAuthFormFields.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1471,6 +1755,9 @@ } }, "src/components/mcp_tools/ToolTestPanel.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 }, @@ -1478,12 +1765,20 @@ "count": 1 } }, + "src/components/mcp_tools/UserEnvVarsModal.tsx": { + "no-nested-ternary": { + "count": 2 + } + }, "src/components/mcp_tools/create_mcp_server.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, "react-hooks/set-state-in-effect": { - "count": 5 + "count": 4 } }, "src/components/mcp_tools/mcp_connect.tsx": { @@ -1495,6 +1790,9 @@ } }, "src/components/mcp_tools/mcp_connection_status.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 } @@ -1515,6 +1813,9 @@ } }, "src/components/mcp_tools/mcp_server_edit.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -1531,6 +1832,9 @@ } }, "src/components/mcp_tools/mcp_servers.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -1544,6 +1848,9 @@ } }, "src/components/mcp_tools/mcp_tools.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -1575,6 +1882,9 @@ } }, "src/components/model_dashboard/HealthCheckComponent.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 }, @@ -1583,6 +1893,9 @@ } }, "src/components/model_dashboard/all_models_table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1591,11 +1904,17 @@ "max-params": { "count": 1 }, + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } }, "src/components/model_dashboard/table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1619,6 +1938,9 @@ } }, "src/components/model_info_view.tsx": { + "no-nested-ternary": { + "count": 14 + }, "no-restricted-imports": { "count": 1 }, @@ -1627,6 +1949,9 @@ } }, "src/components/molecules/filter.tsx": { + "no-nested-ternary": { + "count": 2 + }, "react-hooks/use-memo": { "count": 1 } @@ -1643,6 +1968,9 @@ "max-params": { "count": 1 }, + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } @@ -1656,6 +1984,9 @@ "max-params": { "count": 23 }, + "no-nested-ternary": { + "count": 5 + }, "no-restricted-syntax": { "count": 154 } @@ -1731,6 +2062,9 @@ } }, "src/components/permissions/MCPServerPermissions.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 } @@ -1740,6 +2074,11 @@ "count": 1 } }, + "src/components/policies/PolicySelector.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/policies/add_attachment_form.tsx": { "no-restricted-imports": { "count": 1 @@ -1760,6 +2099,9 @@ } }, "src/components/policies/ai_suggestion_modal.tsx": { + "no-nested-ternary": { + "count": 10 + }, "no-restricted-imports": { "count": 1 }, @@ -1773,11 +2115,17 @@ } }, "src/components/policies/attachment_table.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } }, "src/components/policies/guardrail_selection_modal.tsx": { + "no-nested-ternary": { + "count": 1 + }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1788,6 +2136,9 @@ } }, "src/components/policies/impact_popover.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1806,6 +2157,9 @@ } }, "src/components/policies/pipeline_flow_builder.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -1827,6 +2181,9 @@ } }, "src/components/policies/policy_table.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } @@ -1856,6 +2213,9 @@ } }, "src/components/public_model_hub.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1865,12 +2225,25 @@ "count": 1 } }, + "src/components/router_settings/ReliabilityRetriesSection.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/routing_groups/index.tsx": { "react-hooks/preserve-manual-memoization": { "count": 1 } }, + "src/components/search_tools/SearchToolSelector.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/settings.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } @@ -1925,6 +2298,9 @@ } }, "src/components/team/EditMembership.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 } @@ -1935,6 +2311,9 @@ } }, "src/components/team/TeamInfo.tsx": { + "no-nested-ternary": { + "count": 3 + }, "no-restricted-imports": { "count": 1 }, @@ -1943,6 +2322,9 @@ } }, "src/components/team/TeamVirtualKeysTable.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } @@ -1966,6 +2348,9 @@ } }, "src/components/templates/key_edit_view.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 } @@ -1976,6 +2361,9 @@ } }, "src/components/templates/key_info_view.tsx": { + "no-nested-ternary": { + "count": 1 + }, "no-restricted-imports": { "count": 1 }, @@ -2016,6 +2404,9 @@ } }, "src/components/vector_store_management/VectorStoreForm.tsx": { + "no-nested-ternary": { + "count": 2 + }, "no-restricted-imports": { "count": 1 }, @@ -2044,12 +2435,38 @@ "count": 1 } }, + "src/components/view_logs/EvalViewer/EvalViewer.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/view_logs/GuardrailViewer/CompliancePanel.tsx": { + "no-nested-ternary": { + "count": 2 + }, "react-hooks/set-state-in-effect": { "count": 1 } }, + "src/components/view_logs/GuardrailViewer/ContentFilterDetails.tsx": { + "no-nested-ternary": { + "count": 1 + } + }, + "src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx": { + "no-nested-ternary": { + "count": 4 + } + }, + "src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx": { + "no-nested-ternary": { + "count": 4 + } + }, "src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx": { + "no-nested-ternary": { + "count": 3 + }, "react-hooks/set-state-in-effect": { "count": 2 } @@ -2059,11 +2476,21 @@ "count": 2 } }, + "src/components/view_logs/LogDetailsDrawer/prettyMessagesUtils.ts": { + "no-nested-ternary": { + "count": 1 + } + }, "src/components/view_logs/LogDetailsDrawer/useKeyboardNavigation.ts": { "react-hooks/immutability": { "count": 2 } }, + "src/components/view_logs/LogsTableToolbar.tsx": { + "no-nested-ternary": { + "count": 4 + } + }, "src/components/view_logs/columns.tsx": { "no-restricted-imports": { "count": 1 @@ -2077,6 +2504,11 @@ "count": 1 } }, + "src/components/view_logs/table.tsx": { + "no-nested-ternary": { + "count": 2 + } + }, "src/components/view_user_spend.tsx": { "react-hooks/set-state-in-effect": { "count": 2 @@ -2113,6 +2545,9 @@ } }, "src/hooks/useTestMCPConnection.tsx": { + "no-nested-ternary": { + "count": 1 + }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -2130,6 +2565,11 @@ "count": 1 } }, + "src/lib/http/client.ts": { + "no-nested-ternary": { + "count": 1 + } + }, "src/utils/dataUtils.test.ts": { "max-nested-callbacks": { "count": 1 diff --git a/ui/litellm-dashboard/eslint.config.mjs b/ui/litellm-dashboard/eslint.config.mjs index acdd0c91309..0cf5b4ff655 100644 --- a/ui/litellm-dashboard/eslint.config.mjs +++ b/ui/litellm-dashboard/eslint.config.mjs @@ -3,6 +3,7 @@ import tseslint from "typescript-eslint"; import nextCoreWebVitals from "eslint-config-next/core-web-vitals"; import prettier from "eslint-config-prettier/flat"; import unusedImports from "eslint-plugin-unused-imports"; +import local from "./scripts/eslint-rules/index.mjs"; const eslintConfig = [ { @@ -13,9 +14,11 @@ const eslintConfig = [ ...nextCoreWebVitals, prettier, { - plugins: { "unused-imports": unusedImports }, + plugins: { "unused-imports": unusedImports, local }, rules: { "unused-imports/no-unused-imports": "error", + "local/no-large-inline-object-arg": "warn", + "local/no-long-condition-chain": "warn", "@typescript-eslint/no-explicit-any": "warn", "no-console": ["warn", { allow: ["warn", "error"] }], "@typescript-eslint/no-unused-vars": "off", @@ -28,6 +31,7 @@ const eslintConfig = [ "no-useless-escape": "off", "no-self-assign": "error", "no-var": "error", + "no-nested-ternary": "error", "react/no-danger": "error", complexity: ["warn", 20], "max-depth": ["warn", 4], diff --git a/ui/litellm-dashboard/scripts/eslint-rules/index.mjs b/ui/litellm-dashboard/scripts/eslint-rules/index.mjs new file mode 100644 index 00000000000..150ba1d02e9 --- /dev/null +++ b/ui/litellm-dashboard/scripts/eslint-rules/index.mjs @@ -0,0 +1,11 @@ +import noLargeInlineObjectArg from "./no-large-inline-object-arg.mjs"; +import noLongConditionChain from "./no-long-condition-chain.mjs"; + +const plugin = { + rules: { + "no-large-inline-object-arg": noLargeInlineObjectArg, + "no-long-condition-chain": noLongConditionChain, + }, +}; + +export default plugin; diff --git a/ui/litellm-dashboard/scripts/eslint-rules/no-large-inline-object-arg.mjs b/ui/litellm-dashboard/scripts/eslint-rules/no-large-inline-object-arg.mjs new file mode 100644 index 00000000000..5c5ae170e23 --- /dev/null +++ b/ui/litellm-dashboard/scripts/eslint-rules/no-large-inline-object-arg.mjs @@ -0,0 +1,41 @@ +const DEFAULT_MIN_PROPERTIES = 4; + +const isArgumentOf = (node) => { + const parent = node.parent; + if (parent == null) return false; + if (parent.type !== "CallExpression" && parent.type !== "NewExpression") return false; + return parent.arguments.includes(node); +}; + +const rule = { + meta: { + type: "suggestion", + docs: { + description: + "Disallow passing a large object literal inline as a call argument; assign it to a named variable first.", + }, + schema: [ + { + type: "object", + properties: { minProperties: { type: "integer", minimum: 1 } }, + additionalProperties: false, + }, + ], + messages: { + tooLarge: + "Object literal with {{count}} properties passed inline as an argument; assign it to a named variable first.", + }, + }, + create(context) { + const minProperties = context.options[0]?.minProperties ?? DEFAULT_MIN_PROPERTIES; + return { + ObjectExpression(node) { + if (!isArgumentOf(node)) return; + if (node.properties.length < minProperties) return; + context.report({ node, messageId: "tooLarge", data: { count: node.properties.length } }); + }, + }; + }, +}; + +export default rule; diff --git a/ui/litellm-dashboard/scripts/eslint-rules/no-long-condition-chain.mjs b/ui/litellm-dashboard/scripts/eslint-rules/no-long-condition-chain.mjs new file mode 100644 index 00000000000..638e57442e2 --- /dev/null +++ b/ui/litellm-dashboard/scripts/eslint-rules/no-long-condition-chain.mjs @@ -0,0 +1,41 @@ +const DEFAULT_MIN_CONDITIONS = 4; + +const isBooleanLogical = (node) => + node?.type === "LogicalExpression" && (node.operator === "&&" || node.operator === "||"); + +const countConditions = (node) => + isBooleanLogical(node) ? countConditions(node.left) + countConditions(node.right) : 1; + +const rule = { + meta: { + type: "suggestion", + docs: { + description: + "Disallow logical expressions that combine many conditions; extract the condition into a named boolean.", + }, + schema: [ + { + type: "object", + properties: { minConditions: { type: "integer", minimum: 2 } }, + additionalProperties: false, + }, + ], + messages: { + tooMany: "Boolean expression combines {{count}} conditions; extract it into a named variable.", + }, + }, + create(context) { + const minConditions = context.options[0]?.minConditions ?? DEFAULT_MIN_CONDITIONS; + return { + LogicalExpression(node) { + if (!isBooleanLogical(node)) return; + if (isBooleanLogical(node.parent)) return; + const count = countConditions(node); + if (count < minConditions) return; + context.report({ node, messageId: "tooMany", data: { count } }); + }, + }; + }, +}; + +export default rule; diff --git a/ui/litellm-dashboard/tests/eslint-rules/no-large-inline-object-arg.test.ts b/ui/litellm-dashboard/tests/eslint-rules/no-large-inline-object-arg.test.ts new file mode 100644 index 00000000000..dfe22ea8266 --- /dev/null +++ b/ui/litellm-dashboard/tests/eslint-rules/no-large-inline-object-arg.test.ts @@ -0,0 +1,46 @@ +import { RuleTester } from "eslint"; +import rule from "../../scripts/eslint-rules/no-large-inline-object-arg.mjs"; + +const ruleTester = new RuleTester({ + languageOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +ruleTester.run("no-large-inline-object-arg", rule as never, { + valid: [ + "foo({ a: 1, b: 2, c: 3 });", + "foo({});", + "const opts = { a: 1, b: 2, c: 3, d: 4 }; foo(opts);", + "const x = { a: 1, b: 2, c: 3, d: 4 };", + "function f() { return { a: 1, b: 2, c: 3, d: 4 }; }", + "const arr = [{ a: 1, b: 2, c: 3, d: 4 }];", + "foo(1, 2, { a: 1, b: 2 });", + { code: "foo({ a: 1, b: 2, c: 3, d: 4 });", options: [{ minProperties: 5 }] }, + ], + invalid: [ + { + code: "foo({ a: 1, b: 2, c: 3, d: 4 });", + errors: [{ messageId: "tooLarge", data: { count: 4 } }], + }, + { + code: "new Widget({ a: 1, b: 2, c: 3, d: 4, e: 5 });", + errors: [{ messageId: "tooLarge", data: { count: 5 } }], + }, + { + code: "foo(1, { a: 1, b: 2, c: 3, d: 4 });", + errors: [{ messageId: "tooLarge" }], + }, + { + code: "foo({ a: 1, ...rest, c: 3, d: 4 });", + errors: [{ messageId: "tooLarge", data: { count: 4 } }], + }, + { + code: "foo({ a: 1, b: 2, c: 3 });", + options: [{ minProperties: 3 }], + errors: [{ messageId: "tooLarge", data: { count: 3 } }], + }, + { + code: "outer({ a: 1, b: 2, c: 3, d: 4 }, inner({ e: 5, f: 6, g: 7, h: 8 }));", + errors: [{ messageId: "tooLarge" }, { messageId: "tooLarge" }], + }, + ], +}); diff --git a/ui/litellm-dashboard/tests/eslint-rules/no-long-condition-chain.test.ts b/ui/litellm-dashboard/tests/eslint-rules/no-long-condition-chain.test.ts new file mode 100644 index 00000000000..4a7fb677190 --- /dev/null +++ b/ui/litellm-dashboard/tests/eslint-rules/no-long-condition-chain.test.ts @@ -0,0 +1,51 @@ +import { RuleTester } from "eslint"; +import rule from "../../scripts/eslint-rules/no-long-condition-chain.mjs"; + +const ruleTester = new RuleTester({ + languageOptions: { ecmaVersion: "latest", sourceType: "module" }, +}); + +ruleTester.run("no-long-condition-chain", rule as never, { + valid: [ + "const x = a && b && c;", + "const x = a || b || c;", + "const x = a && (b || c);", + "const x = a && b;", + "if (a || b || c) {}", + "const x = a ?? b ?? c;", + "const url = a ?? b ?? c ?? d;", + "const x = (a && b) ?? c ?? d;", + { code: "const x = a && b && c && d;", options: [{ minConditions: 5 }] }, + ], + invalid: [ + { + code: "const x = a && b && c && d;", + errors: [{ messageId: "tooMany", data: { count: 4 } }], + }, + { + code: "const x = a || b || c || d || e;", + errors: [{ messageId: "tooMany", data: { count: 5 } }], + }, + { + code: "const x = a && b || c && d;", + errors: [{ messageId: "tooMany", data: { count: 4 } }], + }, + { + code: "if (!a && !b && !c && !d) {}", + errors: [{ messageId: "tooMany", data: { count: 4 } }], + }, + { + code: "const x = a && (b || c);", + options: [{ minConditions: 3 }], + errors: [{ messageId: "tooMany", data: { count: 3 } }], + }, + { + code: "const x = (a && b && c && d) || (e && f && g && h);", + errors: [{ messageId: "tooMany", data: { count: 8 } }], + }, + { + code: "const x = (a && b && c && d) ?? fallback;", + errors: [{ messageId: "tooMany", data: { count: 4 } }], + }, + ], +}); From 34aedc40c6467e8a81dbd18e8df71d20cb6bcd96 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Jul 2026 14:50:27 -0700 Subject: [PATCH 024/101] test(ui): mock LicenseExpiryBanner in the dashboard layout test The layout test renders DashboardShell without a QueryClientProvider and mocks DebugWarningBanner to null for exactly that reason. The new LicenseExpiryBanner also uses a React Query hook, so it needs the same treatment; without it the test threw "No QueryClient set". Runtime is unaffected: the app mounts a QueryClientProvider above the layout (DebugWarningBanner already relies on it) --- ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index af68d9f87e9..7573ddb5a0f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -21,6 +21,10 @@ vi.mock("@/components/DebugWarningBanner", () => ({ DebugWarningBanner: () => null, })); +vi.mock("@/components/LicenseExpiryBanner", () => ({ + LicenseExpiryBanner: () => null, +})); + vi.mock("@/contexts/ThemeContext", () => ({ ThemeProvider: ({ children }: { children: React.ReactNode }) => <>{children}, })); From 528fa380f5a271865af9f85228148131063bdf2b Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 8 Jul 2026 15:05:27 -0700 Subject: [PATCH 025/101] fix(guardrails): forward grayswan scan id header (#32544) * fix(guardrails): forward grayswan scan id header * test(guardrails): cover grayswan scan id forwarding * fix(guardrails): prevent overwriting existing metadata headers when extracting scan id * test(guardrails): cover header merging logic * chore(guardrails): fix formatting * test(guardrails): enforce case preservation * chore(guardrails): corrected grayswan type annotations * fix(guardrails): sanitized grayswan header metadata * test(guardrails): covered grayswan logging headers * fix(guardrails): guard grayswan header lookup against None and drop dead comment - Fall back to {} when proxy_server_request is explicitly None so request_data.get(...).get('headers') never raises AttributeError. - Remove the commented-out user_api_key_auth pop; it was inert and greptile called it out as ambiguous. --------- Co-authored-by: Theodore Drzewinski <93957989+tediferJones@users.noreply.github.com> --- .../guardrail_hooks/grayswan/grayswan.py | 48 +++++- .../guardrail_hooks/test_grayswan.py | 147 ++++++++++++++---- 2 files changed, 159 insertions(+), 36 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py index a14d2fc8608..9805b1a9117 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py +++ b/litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py @@ -213,7 +213,7 @@ class GraySwanGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Gray Swan Guardrail: dynamic extra_body=%s", safe_dumps(dynamic_body)) # Prepare and send payload - payload = self._prepare_payload(messages, dynamic_body, request_data) + payload = self._prepare_payload(messages, dynamic_body, request_data, logging_obj) if payload is None: return inputs @@ -502,10 +502,38 @@ class GraySwanGuardrail(CustomGuardrail): "grayswan-api-key": self.api_key, } + def _extract_inbound_headers( + self, + request_data: dict, + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> Optional[dict[str, str]]: + headers = (request_data.get("proxy_server_request") or {}).get("headers") + if not headers: + headers = request_data.get("headers") + if not headers: + headers = (request_data.get("metadata") or {}).get("headers") + if not headers and logging_obj and getattr(logging_obj, "model_call_details", None): + headers = ( + (logging_obj.model_call_details or {}).get("litellm_params", {}).get("metadata", {}).get("headers") + ) + if not isinstance(headers, dict): + return None + + forwarded_header_names = ("shade_scan_id",) + forwarded_headers = {} + for key, value in headers.items(): + if str(key).lower() in forwarded_header_names: + forwarded_headers[str(key)] = str(value) + return forwarded_headers or None + def _prepare_payload( - self, messages: List[Dict[str, str]], dynamic_body: dict, request_data: dict - ) -> Optional[Dict[str, Any]]: - payload: Dict[str, Any] = {"messages": messages} + self, + messages: list[dict[str, str]], + dynamic_body: dict, + request_data: dict, + logging_obj: Optional["LiteLLMLoggingObj"] = None, + ) -> Optional[dict[str, Any]]: + payload: dict[str, Any] = {"messages": messages} categories = dynamic_body.get("categories") or self.categories if categories: @@ -523,10 +551,16 @@ class GraySwanGuardrail(CustomGuardrail): if "metadata" in dynamic_body: payload["metadata"] = dynamic_body["metadata"] + inbound_headers = self._extract_inbound_headers(request_data, logging_obj) + litellm_metadata = request_data.get("litellm_metadata") - if isinstance(litellm_metadata, dict) and litellm_metadata: - cleaned_litellm_metadata = dict(litellm_metadata) - # cleaned_litellm_metadata.pop("user_api_key_auth", None) + cleaned_litellm_metadata = dict(litellm_metadata) if isinstance(litellm_metadata, dict) else {} + if inbound_headers: + existing_headers = cleaned_litellm_metadata.get("headers") + cleaned_litellm_metadata["headers"] = ( + {**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers + ) + if cleaned_litellm_metadata: sanitized = safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={}) if isinstance(sanitized, dict) and sanitized: payload["litellm_metadata"] = sanitized diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py index f2e7447239f..53af7f36a5f 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py @@ -5,6 +5,7 @@ from fastapi import HTTPException from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.guardrail_hooks.grayswan import grayswan as grayswan_module from litellm.proxy.guardrails.guardrail_hooks.grayswan.grayswan import ( GraySwanGuardrail, GraySwanGuardrailAPIError, @@ -70,12 +71,118 @@ def test_prepare_payload_includes_dynamic_metadata( assert payload["metadata"] == dynamic_body["metadata"] +def test_prepare_payload_forwards_only_scan_id_header( + grayswan_guardrail: GraySwanGuardrail, +) -> None: + messages = [{"role": "user", "content": "hello"}] + request_data = { + "proxy_server_request": { + "headers": { + "SHADE_SCAN_ID": "scan-123", + "authorization": "Bearer secret", + } + }, + "litellm_metadata": {"request_id": "request-123"}, + } + + payload = grayswan_guardrail._prepare_payload(messages, {}, request_data) + + assert payload["litellm_metadata"] == { + "request_id": "request-123", + "headers": {"SHADE_SCAN_ID": "scan-123"}, + } + + +def test_prepare_payload_merges_scan_id_with_existing_metadata_headers( + grayswan_guardrail: GraySwanGuardrail, +) -> None: + messages = [{"role": "user", "content": "hello"}] + request_data = { + "proxy_server_request": { + "headers": { + "shade_scan_id": "scan-123", + } + }, + "litellm_metadata": { + "request_id": "request-123", + "headers": {"x-existing": "keep-me"}, + }, + } + + payload = grayswan_guardrail._prepare_payload(messages, {}, request_data) + + assert payload["litellm_metadata"] == { + "request_id": "request-123", + "headers": { + "x-existing": "keep-me", + "shade_scan_id": "scan-123", + }, + } + + +def test_prepare_payload_sanitizes_headers_when_litellm_metadata_absent( + monkeypatch: pytest.MonkeyPatch, + grayswan_guardrail: GraySwanGuardrail, +) -> None: + messages = [{"role": "user", "content": "hello"}] + request_data = { + "proxy_server_request": { + "headers": { + "shade_scan_id": "scan-123", + } + } + } + + monkeypatch.setattr(grayswan_module, "safe_dumps", lambda _data: "{}") + + payload = grayswan_guardrail._prepare_payload(messages, {}, request_data) + + assert "litellm_metadata" not in payload + + +def test_prepare_payload_extracts_headers_from_logging_obj( + grayswan_guardrail: GraySwanGuardrail, +) -> None: + messages = [{"role": "user", "content": "hello"}] + request_data = {} + logging_obj = type( + "LoggingObj", + (), + { + "model_call_details": { + "litellm_params": { + "metadata": { + "headers": { + "shade_scan_id": "scan-from-logging", + "authorization": "Bearer secret", + } + } + } + } + }, + )() + + payload = grayswan_guardrail._prepare_payload(messages, {}, request_data, logging_obj) + + assert payload["litellm_metadata"] == { + "headers": {"shade_scan_id": "scan-from-logging"}, + } + + +def test_prepare_payload_ignores_logging_obj_without_model_call_details( + grayswan_guardrail: GraySwanGuardrail, +) -> None: + messages = [{"role": "user", "content": "hello"}] + + payload = grayswan_guardrail._prepare_payload(messages, {}, {}, object()) + + assert "litellm_metadata" not in payload + + def test_process_response_does_not_block_under_threshold( grayswan_guardrail: GraySwanGuardrail, ) -> None: - grayswan_guardrail._process_grayswan_response( - {"violation": 0.3, "violated_rules": []} - ) + grayswan_guardrail._process_grayswan_response({"violation": 0.3, "violated_rules": []}) def test_process_response_blocks_when_threshold_exceeded() -> None: @@ -127,16 +234,12 @@ class _DummyClient: self.calls: list[dict] = [] async def post(self, *, url: str, headers: dict, json: dict, timeout: float): - self.calls.append( - {"url": url, "headers": headers, "json": json, "timeout": timeout} - ) + self.calls.append({"url": url, "headers": headers, "json": json, "timeout": timeout}) return _DummyResponse(self.payload) @pytest.mark.asyncio -async def test_run_guardrail_posts_payload( - monkeypatch, grayswan_guardrail: GraySwanGuardrail -) -> None: +async def test_run_guardrail_posts_payload(monkeypatch, grayswan_guardrail: GraySwanGuardrail) -> None: dummy_client = _DummyClient({"violation": 0.1}) grayswan_guardrail.async_handler = dummy_client @@ -308,9 +411,7 @@ def test_process_response_passthrough_raises_exception_in_pre_call() -> None: # Should raise ModifyResponseException with pytest.raises(ModifyResponseException) as exc: - guardrail._process_grayswan_response( - response_json, data, GuardrailEventHooks.pre_call - ) + guardrail._process_grayswan_response(response_json, data, GuardrailEventHooks.pre_call) assert "Gray Swan Cygnal Guardrail" in exc.value.message assert exc.value.model == "gpt-4" @@ -338,9 +439,7 @@ def test_process_response_passthrough_raises_exception_in_during_call() -> None: # Should raise ModifyResponseException with pytest.raises(ModifyResponseException) as exc: - guardrail._process_grayswan_response( - response_json, data, GuardrailEventHooks.during_call - ) + guardrail._process_grayswan_response(response_json, data, GuardrailEventHooks.during_call) assert "Gray Swan Cygnal Guardrail" in exc.value.message assert exc.value.model == "gpt-4" @@ -365,9 +464,7 @@ def test_process_response_passthrough_stores_detection_info_in_post_call() -> No } # Should NOT raise an exception in post_call - guardrail._process_grayswan_response( - response_json, data, GuardrailEventHooks.post_call - ) + guardrail._process_grayswan_response(response_json, data, GuardrailEventHooks.post_call) # Verify detection info was stored in metadata assert "metadata" in data @@ -400,9 +497,7 @@ def test_process_response_passthrough_does_not_raise_if_under_threshold() -> Non } # Should not raise an exception since under threshold - guardrail._process_grayswan_response( - response_json, data, GuardrailEventHooks.pre_call - ) + guardrail._process_grayswan_response(response_json, data, GuardrailEventHooks.pre_call) # Should not have any detection info since it didn't exceed threshold assert "guardrail_detections" not in data.get("metadata", {}) @@ -436,10 +531,7 @@ def test_format_violation_message() -> None: assert "Gray Swan Cygnal Guardrail" in message assert "the input query has a violation score of 0.85" in message assert "violating the rule(s): 1, 3, 5" in message - assert ( - "Mutation effort to make the harmful intention disguised was DETECTED" - in message - ) + assert "Mutation effort to make the harmful intention disguised was DETECTED" in message # IPI should not be in message since it's False assert "Indirect Prompt Injection was DETECTED" not in message @@ -450,10 +542,7 @@ def test_format_violation_message() -> None: assert "Gray Swan Cygnal Guardrail" in message assert "the model response has a violation score of 0.85" in message assert "violating the rule(s): 1, 3, 5" in message - assert ( - "Mutation effort to make the harmful intention disguised was DETECTED" - in message - ) + assert "Mutation effort to make the harmful intention disguised was DETECTED" in message def test_prepare_payload_includes_litellm_metadata( From 641396762ac8e363325ae1e177e8079e4edec9e4 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Jul 2026 15:17:13 -0700 Subject: [PATCH 026/101] refactor(ui): conform license banner to new eslint rules Staging recently added the local eslint rules no-large-inline-object-arg and no-long-condition-chain and tightened no-nested-ternary to an error. After merging staging, the license-banner code tripped them: the banner's tiered description was a nested ternary (now an error), and two option objects were passed inline (adding budget debt). Extract the description into an early-return helper, and hoist the useQuery options and the date-format options into named constants. No behavior change; keeps the inline-object-arg count at the committed baseline rather than bumping it --- .../hooks/license/useLicenseInfo.ts | 5 +++-- .../src/components/LicenseExpiryBanner.tsx | 19 +++++++++++-------- .../src/utils/licenseUtils.ts | 14 ++++++++------ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts index f4574c36ef6..3ea0bd20e40 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/license/useLicenseInfo.ts @@ -5,11 +5,12 @@ import { createQueryKeys } from "../common/queryKeysFactory"; const licenseInfoKeys = createQueryKeys("licenseInfo"); export const useLicenseInfo = (accessToken: string | null | undefined): UseQueryResult => { - return useQuery({ + const options = { queryKey: licenseInfoKeys.detail("license"), queryFn: () => getLicenseInfo(accessToken!), enabled: Boolean(accessToken), staleTime: 5 * 60 * 1000, retry: false, - }); + }; + return useQuery(options); }; diff --git a/ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx b/ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx index e5b8a65168a..c3b20b5fac0 100644 --- a/ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx +++ b/ui/litellm-dashboard/src/components/LicenseExpiryBanner.tsx @@ -29,6 +29,16 @@ const describeCountdown = (days: number): string => { return `expires in ${days} days`; }; +const expiryDescription = (tier: "warning" | "critical" | "expired"): React.ReactNode => { + if (tier === "expired") { + return <>Enterprise features are now disabled. Reach out to {salesLink} to restore access; + } + if (tier === "critical") { + return <>Renew now to avoid losing enterprise features. Reach out to {salesLink}; + } + return <>Renew before it lapses to keep enterprise features. Reach out to {salesLink}; +}; + export const LicenseExpiryBannerView: React.FC = ({ licenseInfo }) => { const [locallyDismissed, setLocallyDismissed] = useState(false); @@ -56,14 +66,7 @@ export const LicenseExpiryBannerView: React.FC = ( ? `Your LiteLLM Enterprise license expired on ${formattedDate}` : `Your LiteLLM Enterprise license ${describeCountdown(days)} (${formattedDate})`; - const description = - tier === "expired" ? ( - <>Enterprise features are now disabled. Reach out to {salesLink} to restore access - ) : tier === "critical" ? ( - <>Renew now to avoid losing enterprise features. Reach out to {salesLink} - ) : ( - <>Renew before it lapses to keep enterprise features. Reach out to {salesLink} - ); + const description = expiryDescription(tier); const handleClose = () => { if (typeof window !== "undefined") { diff --git a/ui/litellm-dashboard/src/utils/licenseUtils.ts b/ui/litellm-dashboard/src/utils/licenseUtils.ts index b2681664c56..57acad85508 100644 --- a/ui/litellm-dashboard/src/utils/licenseUtils.ts +++ b/ui/litellm-dashboard/src/utils/licenseUtils.ts @@ -35,15 +35,17 @@ export const getLicenseExpiryTier = (expirationDate: string | null, now: Date = return "none"; }; +const EXPIRY_DATE_FORMAT: Intl.DateTimeFormatOptions = { + year: "numeric", + month: "short", + day: "numeric", + timeZone: "UTC", +}; + export const formatExpiryDate = (expirationDate: string): string => { const date = new Date(`${expirationDate}T00:00:00Z`); if (Number.isNaN(date.getTime())) { return expirationDate; } - return date.toLocaleDateString("en-US", { - year: "numeric", - month: "short", - day: "numeric", - timeZone: "UTC", - }); + return date.toLocaleDateString("en-US", EXPIRY_DATE_FORMAT); }; From bd23c44cb197e71143c4bae838b8af0da1869971 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 8 Jul 2026 15:28:28 -0700 Subject: [PATCH 027/101] refactor(ui): consolidate table cells onto a shared table_cells kit (#32393) * feat(ui): add shared table_cells kit and convert logs columns DateCell, MoneyCell, IdCell and StatusBadge consolidate the duplicated per-table cell implementations behind one component each. The logs page columns are the reference conversion; the dead auditLogColumns export (superseded by audit_logs.tsx) is removed with it * refactor(ui): consolidate table cells onto the shared table_cells kit 106 cell sites across 44 table files converge onto DateCell, MoneyCell, IdCell and StatusBadge, replacing 8 date formats, 6 spend formats, 7 id truncation strategies and 6 status badge styles with one implementation each. Badge now forwards refs so Base UI tooltip triggers composed over it can attach (they previously never opened under React 18). TimeCell is deleted; its two consumers now render DateCell * fix(ui): suppress cost tooltip for zero spend and drop dead getStatusBadge param The logs Cost tooltip showed the raw $0 over a "-" cell for zero or null spend (pre-existing, surfaced by review); the tooltip now only renders when there is a real amount. healthCheckColumns no longer takes the unused getStatusBadge callback and its dead definition is removed * fix(ui): restyle StatusBadge as tinted pill matching the prior antd Tag look * fix(ui): keep StatusBadge fully rounded like the other kit pills --- ui/litellm-dashboard/eslint-metrics.json | 4 +- ui/litellm-dashboard/eslint-suppressions.json | 7 +- .../components/AccessGroupsPage.tsx | 21 +- .../budgets/components/budget_panel.tsx | 5 +- .../memory/components/MemoryView.tsx | 32 +-- .../projects/components/ProjectKeysTable.tsx | 5 +- .../projects/components/ProjectsPage.tsx | 18 +- .../prompts/components/prompt_table.tsx | 59 +---- .../_components/SearchToolColumn.tsx | 14 +- .../users/_components/BulkEditUsers.test.tsx | 2 +- .../users/_components/BulkEditUsers.tsx | 3 +- .../users/_components/view_users.test.tsx | 3 +- .../users/_components/view_users/columns.tsx | 42 +--- .../components/AIHub/AgentHubTableColumns.tsx | 13 +- .../DeletedKeysTable/DeletedKeysTable.tsx | 49 +---- .../DeletedTeamsTable/DeletedTeamsTable.tsx | 50 +---- .../src/components/OldTeams.tsx | 23 +- .../LoggingCallbacksTable.tsx | 14 +- .../src/components/ToolPolicies.tsx | 14 +- .../components/EndpointUsageTable.test.tsx | 6 - .../components/EndpointUsageTable.tsx | 4 +- .../components/EntityUsage/EntityUsage.tsx | 9 +- .../EntityUsage/SpendByProvider.test.tsx | 10 +- .../EntityUsage/SpendByProvider.tsx | 5 +- .../EntityUsage/TopKeyView.test.tsx | 27 +-- .../components/EntityUsage/TopKeyView.tsx | 23 +- .../EntityUsage/TopModelView.test.tsx | 2 +- .../components/EntityUsage/TopModelView.tsx | 6 +- .../components/KeyModelUsageView.test.tsx | 2 +- .../components/KeyModelUsageView.tsx | 3 +- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 10 +- .../VirtualKeysPage/VirtualKeysTable.tsx | 79 ++----- .../src/components/agents.tsx | 21 +- .../claude_code_plugins/plugin_table.tsx | 34 +-- .../src/components/general_settings.tsx | 12 +- .../components/guardrails/guardrail_table.tsx | 52 +---- .../src/components/mcp_hub_table_columns.tsx | 21 +- .../components/mcp_tools/MCPToolsetsTab.tsx | 13 +- .../model_dashboard/HealthCheckComponent.tsx | 18 +- .../model_dashboard/health_check_columns.tsx | 25 ++- .../components/model_hub_table_columns.tsx | 9 +- .../molecules/models/columns.test.tsx | 58 +++++ .../components/molecules/models/columns.tsx | 54 ++--- .../organization/organization_view.tsx | 3 +- .../src/components/organizations.tsx | 32 ++- .../src/components/pass_through_settings.tsx | 16 +- .../policies/attachment_table.test.tsx | 7 +- .../components/policies/attachment_table.tsx | 25 +-- .../src/components/policies/policy_table.tsx | 16 +- .../shared/table_cells/cell_tooltip.tsx | 21 ++ .../shared/table_cells/date_cell.test.tsx | 57 +++++ .../shared/table_cells/date_cell.tsx | 41 ++++ .../shared/table_cells/id_cell.test.tsx | 78 +++++++ .../components/shared/table_cells/id_cell.tsx | 94 ++++++++ .../components/shared/table_cells/index.ts | 5 + .../shared/table_cells/money_cell.test.tsx | 44 ++++ .../shared/table_cells/money_cell.tsx | 23 ++ .../shared/table_cells/status_badge.test.tsx | 42 ++++ .../shared/table_cells/status_badge.tsx | 38 ++++ .../components/skill_hub_table_columns.tsx | 8 +- .../tag_management/TagTable.test.tsx | 18 +- .../components/tag_management/TagTable.tsx | 39 +--- .../components/team/TeamMemberTab.test.tsx | 27 ++- .../src/components/team/TeamMemberTab.tsx | 36 +--- .../components/team/TeamVirtualKeysTable.tsx | 76 ++----- .../src/components/ui/badge.tsx | 22 +- ui/litellm-dashboard/src/components/usage.tsx | 9 +- .../DocumentsTable.tsx | 17 +- .../VectorStoreTable.test.tsx | 9 +- .../VectorStoreTable.tsx | 25 +-- .../src/components/view_logs/audit_logs.tsx | 12 +- .../src/components/view_logs/columns.test.tsx | 56 +++++ .../src/components/view_logs/columns.tsx | 202 ++---------------- .../components/view_logs/time_cell.test.tsx | 36 ---- .../src/components/view_logs/time_cell.tsx | 42 ---- 75 files changed, 918 insertions(+), 1139 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/cell_tooltip.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/date_cell.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/date_cell.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/index.ts create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/money_cell.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/money_cell.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/status_badge.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/table_cells/status_badge.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/columns.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_logs/time_cell.tsx diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index f4dc89c5b80..ded6ab97e1e 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,7 +1,7 @@ { - "@typescript-eslint/no-explicit-any": 1988, + "@typescript-eslint/no-explicit-any": 1982, "complexity": 128, - "local/no-large-inline-object-arg": 513, + "local/no-large-inline-object-arg": 512, "local/no-long-condition-chain": 233, "max-depth": 59, "no-console": 15 diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index b077338c75b..fe8f182c106 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1650,7 +1650,7 @@ "count": 1 }, "no-restricted-imports": { - "count": 2 + "count": 1 } }, "src/components/guardrails/tool_permission/ToolPermissionRulesEditor.tsx": { @@ -2491,11 +2491,6 @@ "count": 4 } }, - "src/components/view_logs/columns.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/view_logs/index.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.tsx index c3dd1d54b32..dbbf4e35900 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.tsx @@ -19,6 +19,7 @@ import { SortState, TableHeaderSortDropdown, } from "@/components/common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { AccessGroupDetail } from "./AccessGroupsDetailsPage"; import { AccessGroupCreateModal } from "./AccessGroupsModal/AccessGroupCreateModal"; import { AccessGroup } from "./types"; @@ -143,21 +144,7 @@ export function AccessGroupsPage() { header: () => ID, enableSorting: false, size: 170, - cell: ({ row }) => { - const record = row.original; - return ( - - setSelectedGroupId(record.id)} - > - {record.id} - - - ); - }, + cell: ({ row }) => , }, { id: "name", @@ -211,7 +198,7 @@ export function AccessGroupsPage() { header: () => Created, enableSorting: true, sortingFn: "datetime", - cell: ({ getValue }) => new Date(getValue() as string).toLocaleDateString(), + cell: ({ getValue }) => , meta: { responsive: ["lg"] }, }, { @@ -219,7 +206,7 @@ export function AccessGroupsPage() { accessorKey: "updatedAt", header: () => Updated, enableSorting: false, - cell: ({ getValue }) => new Date(getValue() as string).toLocaleDateString(), + cell: ({ getValue }) => , meta: { responsive: ["xl"] }, }, ...(canModify diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.tsx index 0e601645c20..af15a99f0b4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.tsx @@ -25,6 +25,7 @@ import DeleteResourceModal from "@/components/common_components/DeleteResourceMo import TableIconActionButton from "@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { useBudgets, useDeleteBudget, budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import { MoneyCell } from "@/components/shared/table_cells"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; import { CREATE_END_USER_CURL_COMMAND, CHAT_COMPLETIONS_CURL_COMMAND, OPENAI_SDK_PYTHON_CODE } from "./constants"; @@ -127,7 +128,9 @@ const BudgetPanel: React.FC = ({ accessToken }) => { .map((value: budgetItem) => ( {value.budget_id} - {value.max_budget ? value.max_budget : "n/a"} + + + {value.tpm_limit ? value.tpm_limit : "n/a"} {value.rpm_limit ? value.rpm_limit : "n/a"} {canModify && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryView.tsx index 402de29e0c5..4ee784f4664 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryView.tsx @@ -2,7 +2,7 @@ import React, { useMemo, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Button, Card, Drawer, Empty, Input, Space, Table, Tooltip, Typography, message } from "antd"; +import { Button, Card, Drawer, Empty, Input, Space, Table, Typography, message } from "antd"; import type { ColumnsType } from "antd/es/table"; import { DeleteOutlined, @@ -13,6 +13,7 @@ import { SearchOutlined, } from "@ant-design/icons"; import { MemoryRow, createMemory, deleteMemory, fetchMemoryList, updateMemory } from "@/components/networking"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { MemoryEditModal } from "./MemoryEditModal"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; @@ -191,34 +192,13 @@ export const MemoryView: React.FC = ({ accessToken }) => { } }; - const renderIdPill = (id: string | null | undefined, onClick?: () => void) => { - if (!id) return -; - const short = id.length > 10 ? `${id.slice(0, 7)}...` : id; - const pillClass = - "font-mono text-blue-600 bg-blue-50 text-xs font-medium px-2 py-0.5 rounded-md border border-blue-200 inline-block max-w-[15ch] truncate whitespace-nowrap"; - return ( - - {onClick ? ( - - ) : ( - {short} - )} - - ); - }; - const columns: ColumnsType = [ { title: "ID", dataIndex: "memory_id", key: "memory_id", width: 140, - render: (_: unknown, r: MemoryRow) => renderIdPill(r.memory_id, () => setDetailRow(r)), + render: (_: unknown, r: MemoryRow) => setDetailRow(r)} />, }, { title: "Name", @@ -246,21 +226,21 @@ export const MemoryView: React.FC = ({ accessToken }) => { dataIndex: "user_id", key: "user_id", width: 160, - render: (uid?: string | null) => renderIdPill(uid), + render: (uid?: string | null) => , }, { title: "Team ID", dataIndex: "team_id", key: "team_id", width: 160, - render: (tid?: string | null) => renderIdPill(tid), + render: (tid?: string | null) => , }, { title: "Updated", dataIndex: "updated_at", key: "updated_at", width: 180, - render: (ts?: string) => {formatTimestamp(ts)}, + render: (ts?: string) => , // No sorter — backend already returns rows in `updated_at DESC` order, // and a client-side sorter on a paginated view would only affect the // current page. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.tsx index 7b891078d36..8269c843b98 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.tsx @@ -3,6 +3,7 @@ import { Empty, Table, Tooltip } from "antd"; import type { ColumnsType } from "antd/es/table"; import type { SpinProps } from "antd"; import DefaultProxyAdminTag from "@/components/common_components/DefaultProxyAdminTag"; +import { DateCell } from "@/components/shared/table_cells"; interface ProjectKeysTableProps { keys: KeyResponse[]; @@ -33,13 +34,13 @@ const columns: ColumnsType = [ title: "Created", dataIndex: "created_at", key: "created_at", - render: (date: string) => (date ? new Date(date).toLocaleDateString() : "—"), + render: (date: string) => , }, { title: "Last Active", dataIndex: "last_active", key: "last_active", - render: (date: string | null) => (date ? new Date(date).toLocaleDateString() : "Never"), + render: (date: string | null) => , }, ]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.tsx index 8a3fc178914..be989229022 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.tsx @@ -1,5 +1,6 @@ import { useProjects, ProjectResponse } from "@/app/(dashboard)/hooks/projects/useProjects"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { LoadingOutlined, PlusOutlined } from "@ant-design/icons"; import { Button, @@ -72,18 +73,7 @@ export function ProjectsPage() { dataIndex: "project_id", key: "project_id", width: 170, - render: (id: string) => ( - - setSelectedProjectId(id)} - > - {id} - - - ), + render: (id: string) => , }, { title: "Name", @@ -137,14 +127,14 @@ export function ProjectsPage() { key: "created_at", sorter: (a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime(), responsive: ["lg"], - render: (date: string) => new Date(date).toLocaleDateString(), + render: (date: string) => , }, { title: "Updated", dataIndex: "updated_at", key: "updated_at", responsive: ["xl"], - render: (date: string) => new Date(date).toLocaleDateString(), + render: (date: string) => , }, ]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/components/prompt_table.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/components/prompt_table.tsx index 94b242ca4ea..51a03d19e83 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/components/prompt_table.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/components/prompt_table.tsx @@ -2,8 +2,8 @@ import React, { useState, useEffect } from "react"; import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Button } from "@tremor/react"; import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon, TrashIcon } from "@heroicons/react/outline"; import { Tooltip } from "antd"; -import { CopyOutlined } from "@ant-design/icons"; import { PromptSpec, modelHubCall } from "@/components/networking"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { ColumnDef, flexRender, @@ -62,48 +62,11 @@ const PromptTable: React.FC = ({ fetchModelHubData(); }, [accessToken]); - // Format date helper function - const formatDate = (dateString?: string) => { - if (!dateString) return "-"; - const date = new Date(dateString); - return date.toLocaleString(); - }; - - const copyToClipboard = (text: string) => { - navigator.clipboard.writeText(text); - }; - const columns: ColumnDef[] = [ { header: "Prompt ID", accessorKey: "prompt_id", - cell: (info: any) => { - const fullId = String(info.getValue() || ""); - const displayId = fullId.length > 25 ? `${fullId.slice(0, 25)}...` : fullId; - return ( -
- - - - - { - e.stopPropagation(); - copyToClipboard(fullId); - }} - className="cursor-pointer text-gray-500 hover:text-blue-500 text-xs" - /> - -
- ); - }, + cell: (info: any) => , }, { header: "Model", @@ -162,26 +125,12 @@ const PromptTable: React.FC = ({ { header: "Created At", accessorKey: "created_at", - cell: ({ row }) => { - const prompt = row.original; - return ( - - {formatDate(prompt.created_at)} - - ); - }, + cell: ({ row }) => , }, { header: "Updated At", accessorKey: "updated_at", - cell: ({ row }) => { - const prompt = row.original; - return ( - - {formatDate(prompt.updated_at)} - - ); - }, + cell: ({ row }) => , }, { header: "Environment", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolColumn.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolColumn.tsx index 198b3ea095f..3d9fdb2866e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolColumn.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchToolColumn.tsx @@ -1,6 +1,7 @@ import { Tag } from "antd"; import { ColumnsType } from "antd/es/table"; import TableIconActionButton from "@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { SearchTool } from "./types"; export const searchToolColumns = ( @@ -20,14 +21,7 @@ export const searchToolColumns = ( return -; } - return ( - - ); + return ; }, }, { @@ -52,7 +46,7 @@ export const searchToolColumns = ( dataIndex: "created_at", key: "created_at", render: (_, tool) => { - return {tool.created_at ? new Date(tool.created_at).toLocaleDateString() : "-"}; + return ; }, }, { @@ -60,7 +54,7 @@ export const searchToolColumns = ( dataIndex: "updated_at", key: "updated_at", render: (_, tool) => { - return {tool.updated_at ? new Date(tool.updated_at).toLocaleDateString() : "-"}; + return ; }, }, { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx index f16f5325952..e49ac854e6d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.test.tsx @@ -91,7 +91,7 @@ describe("BulkEditUserModal", () => { it("should display budget information in table", () => { renderWithProviders(); - expect(screen.getByText("$50")).toBeInTheDocument(); + expect(screen.getByText("$50.00")).toBeInTheDocument(); expect(screen.getByText("Unlimited")).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx index 9bef2f3a937..7ea53352807 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx @@ -4,6 +4,7 @@ import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "@/compone import { UserEditView } from "./user_edit_view"; import NotificationsManager from "@/components/molecules/notifications_manager"; import MessageManager from "@/components/molecules/message_manager"; +import { MoneyCell } from "@/components/shared/table_cells"; const { Text, Title } = Typography; @@ -270,7 +271,7 @@ const BulkEditUserModal: React.FC = ({ key: "max_budget", width: "20%", render: (budget: number | null) => ( - {budget !== null ? `$${budget}` : "Unlimited"} + ), }, ]} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx index 241cda3464e..996fd58efc1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users.test.tsx @@ -158,7 +158,8 @@ describe("ViewUserDashboard", () => { expect( screen.getByText("Are you sure you want to delete this user? This action cannot be undone."), ).toBeInTheDocument(); - expect(screen.getByText("user-1")).toBeInTheDocument(); + const userIdInstances = screen.getAllByText("user-1"); + expect(userIdInstances.length).toBeGreaterThan(0); const emailInstances = screen.getAllByText("test@example.com"); expect(emailInstances.length).toBeGreaterThan(0); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx index deeaeeb25f4..fc680cb5b1b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/columns.tsx @@ -3,8 +3,7 @@ import { Badge, Grid, Icon } from "@tremor/react"; import { Tooltip, Checkbox, Tag } from "antd"; import { UserInfo } from "@/components/networking"; import { PencilAltIcon, TrashIcon, InformationCircleIcon, RefreshIcon } from "@heroicons/react/outline"; -import { CopyOutlined } from "@ant-design/icons"; -import { formatNumberWithCommas, copyToClipboard } from "@/utils/dataUtils"; +import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; interface SelectionOptions { selectedUsers: UserInfo[]; @@ -29,24 +28,7 @@ export const columns = ( header: "User ID", accessorKey: "user_id", enableSorting: true, - cell: ({ row }) => ( -
- - {row.original.user_id ? `${row.original.user_id.slice(0, 7)}...` : "-"} - - {row.original.user_id && ( - - { - e.stopPropagation(); - copyToClipboard(row.original.user_id, "User ID copied to clipboard"); - }} - className="cursor-pointer text-gray-500 hover:text-blue-500 text-xs" - /> - - )} -
- ), + cell: ({ row }) => , }, { header: "Email", @@ -93,17 +75,13 @@ export const columns = ( header: "Spend (USD)", accessorKey: "spend", enableSorting: true, - cell: ({ row }) => ( - {row.original.spend ? formatNumberWithCommas(row.original.spend, 4) : "-"} - ), + cell: ({ row }) => , }, { header: "Budget (USD)", accessorKey: "max_budget", enableSorting: false, - cell: ({ row }) => ( - {row.original.max_budget !== null ? row.original.max_budget : "Unlimited"} - ), + cell: ({ row }) => , }, { header: () => ( @@ -142,21 +120,13 @@ export const columns = ( header: "Created At", accessorKey: "created_at", enableSorting: true, - cell: ({ row }) => ( - - {row.original.created_at ? new Date(row.original.created_at).toLocaleDateString() : "-"} - - ), + cell: ({ row }) => , }, { header: "Updated At", accessorKey: "updated_at", enableSorting: false, - cell: ({ row }) => ( - - {row.original.updated_at ? new Date(row.original.updated_at).toLocaleDateString() : "-"} - - ), + cell: ({ row }) => , }, { id: "actions", diff --git a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx index 09b1c147615..ae1a19ff95d 100644 --- a/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/AgentHubTableColumns.tsx @@ -2,6 +2,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Button, Badge, Text } from "@tremor/react"; import { Tooltip, Tag } from "antd"; import { CopyOutlined, InfoCircleOutlined } from "@ant-design/icons"; +import { StatusBadge } from "@/components/shared/table_cells"; export interface AgentHubData { agent_id?: string; @@ -194,17 +195,9 @@ export const getAgentHubTableColumns = ( return publicA - publicB; }, cell: ({ row }) => { - const agent = row.original; + const isPublic = row.original.is_public === true; - return agent.is_public === true ? ( - - Yes - - ) : ( - - No - - ); + return ; }, meta: { className: "hidden md:table-cell", diff --git a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx index bc52bbbe062..d4a120d0589 100644 --- a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx @@ -1,5 +1,5 @@ "use client"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; import { ChevronDownIcon, ChevronUpIcon, SwitchVerticalIcon } from "@heroicons/react/outline"; import { ColumnDef, @@ -59,14 +59,7 @@ export function DeletedKeysTable({ header: "Key ID", size: 150, maxSize: 250, - cell: (info) => { - const value = info.getValue() as string; - return ( - - {value || "-"} - - ); - }, + cell: (info) => , }, { id: "key_alias", @@ -100,9 +93,7 @@ export function DeletedKeysTable({ header: "Spend (USD)", size: 100, maxSize: 140, - cell: (info) => ( - {formatNumberWithCommas(info.getValue() as number, 4)} - ), + cell: (info) => , }, { id: "max_budget", @@ -110,14 +101,9 @@ export function DeletedKeysTable({ header: "Budget (USD)", size: 110, maxSize: 150, - cell: (info) => { - const maxBudget = info.getValue() as number | null; - return ( - - {maxBudget === null ? "Unlimited" : `$${formatNumberWithCommas(maxBudget)}`} - - ); - }, + cell: (info) => ( + + ), }, { id: "user_email", @@ -140,14 +126,7 @@ export function DeletedKeysTable({ header: "User ID", size: 120, maxSize: 200, - cell: (info) => { - const userId = info.getValue() as string | null; - return ( - - {userId || "-"} - - ); - }, + cell: (info) => , }, { id: "created_at", @@ -155,12 +134,7 @@ export function DeletedKeysTable({ header: "Created At", size: 120, maxSize: 140, - cell: (info) => { - const value = info.getValue(); - return ( - {value ? new Date(value as string).toLocaleDateString() : "-"} - ); - }, + cell: (info) => , }, { id: "created_by", @@ -183,10 +157,9 @@ export function DeletedKeysTable({ header: "Deleted At", size: 120, maxSize: 140, - cell: (info) => { - const value = (info.row.original as any).deleted_at as string | null | undefined; - return {value ? new Date(value).toLocaleDateString() : "-"}; - }, + cell: (info) => ( + + ), }, { id: "deleted_by", diff --git a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx index 57260f6bb2d..ddfd5cf73b6 100644 --- a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx +++ b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx @@ -1,5 +1,5 @@ "use client"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; import { ChevronDownIcon, ChevronUpIcon, SwitchVerticalIcon } from "@heroicons/react/outline"; import { ColumnDef, @@ -51,14 +51,7 @@ export function DeletedTeamsTable({ teams, isLoading, isFetching }: DeletedTeams header: "Team ID", size: 150, maxSize: 250, - cell: (info) => { - const value = info.getValue() as string; - return ( - - {value || "-"} - - ); - }, + cell: (info) => , }, { id: "created_at", @@ -66,12 +59,7 @@ export function DeletedTeamsTable({ teams, isLoading, isFetching }: DeletedTeams header: "Created", size: 120, maxSize: 140, - cell: (info) => { - const value = info.getValue(); - return ( - {value ? new Date(value as string).toLocaleDateString() : "-"} - ); - }, + cell: (info) => , }, { id: "spend", @@ -79,12 +67,7 @@ export function DeletedTeamsTable({ teams, isLoading, isFetching }: DeletedTeams header: "Spend (USD)", size: 100, maxSize: 140, - cell: (info) => { - const spend = (info.row.original as any).spend as number | undefined; - return ( - {spend !== undefined ? formatNumberWithCommas(spend, 4) : "-"} - ); - }, + cell: (info) => , }, { id: "max_budget", @@ -92,14 +75,9 @@ export function DeletedTeamsTable({ teams, isLoading, isFetching }: DeletedTeams header: "Budget (USD)", size: 110, maxSize: 150, - cell: (info) => { - const maxBudget = info.getValue() as number | null; - return ( - - {maxBudget === null || maxBudget === undefined ? "No limit" : `$${formatNumberWithCommas(maxBudget)}`} - - ); - }, + cell: (info) => ( + + ), }, { id: "models", @@ -148,14 +126,7 @@ export function DeletedTeamsTable({ teams, isLoading, isFetching }: DeletedTeams header: "Organization", size: 150, maxSize: 200, - cell: (info) => { - const value = info.getValue() as string; - return ( - - {value || "-"} - - ); - }, + cell: (info) => , }, { id: "deleted_at", @@ -163,10 +134,7 @@ export function DeletedTeamsTable({ teams, isLoading, isFetching }: DeletedTeams header: "Deleted At", size: 120, maxSize: 140, - cell: (info) => { - const value = (info.row.original as any).deleted_at as string | null | undefined; - return {value ? new Date(value).toLocaleDateString() : "-"}; - }, + cell: (info) => , }, { id: "deleted_by", diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index c2cb3d4948e..e83d1acf4e5 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -31,6 +31,7 @@ import type { SorterResult } from "antd/es/table/interface"; import { KeyIcon, LayersIcon, SearchIcon, UsersIcon } from "lucide-react"; import React, { useEffect, useMemo, useRef, useState } from "react"; import { AntDLoadingSpinner } from "@/components/ui/AntDLoadingSpinner"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import OrganizationDropdown from "./common_components/OrganizationDropdown"; import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import { teamListCall as v2TeamListCall, type TeamsResponse } from "@/app/(dashboard)/hooks/teams/useTeams"; @@ -670,18 +671,8 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser key: "team_id", width: 170, ellipsis: true, - render: (id: string, record: Team) => ( - - setSelectedTeamId(record.team_id)} - data-testid="team-id-cell" - > - {id} - - + render: (id: string) => ( + setSelectedTeamId(teamId)} dataTestId="team-id-cell" /> ), }, { @@ -797,13 +788,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser width: 130, ellipsis: true, sorter: true, - render: (date: string | undefined) => ( - - {date - ? new Date(date).toLocaleDateString(undefined, { year: "numeric", month: "short", day: "numeric" }) - : "—"} - - ), + render: (date: string | undefined) => , }, { title: "Actions", diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 70ec6599ca2..4f1889cbc99 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -3,6 +3,7 @@ import type { TableProps } from "antd"; import { Table } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; +import { StatusBadge, type StatusTone } from "@/components/shared/table_cells"; import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import { AlertingObject } from "./types"; @@ -61,17 +62,8 @@ export const LoggingCallbacksTable: React.FC = ({ // and server-fetched rows both render correctly. const mode = record.type || record.mode || "success"; const label = CALLBACK_MODES.find((m) => m.value === mode)?.label || mode; - const badgeClass = - mode === "success" - ? "bg-green-100 text-green-800" - : mode === "failure" - ? "bg-red-100 text-red-800" - : "bg-blue-100 text-blue-800"; - return ( - - {label} - - ); + const tone: StatusTone = mode === "success" ? "success" : mode === "failure" ? "error" : "info"; + return ; }, width: 240, }, diff --git a/ui/litellm-dashboard/src/components/ToolPolicies.tsx b/ui/litellm-dashboard/src/components/ToolPolicies.tsx index 4bd028f0c8f..4468334f813 100644 --- a/ui/litellm-dashboard/src/components/ToolPolicies.tsx +++ b/ui/litellm-dashboard/src/components/ToolPolicies.tsx @@ -3,7 +3,7 @@ import React, { useCallback, useDeferredValue, useEffect, useMemo, useState } from "react"; import { Button, Switch, Tooltip } from "antd"; import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; -import { TimeCell } from "./view_logs/time_cell"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import type { SortState } from "./common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; import { TableHeaderSortDropdown } from "./common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; import FilterComponent, { FilterOption } from "./molecules/filter"; @@ -461,7 +461,7 @@ export const ToolPolicies: React.FC = ({ accessToken, onSelec paginated.map((tool) => ( - +
- - {tool.team_id ?? "-"} - + - - - {tool.key_hash ?? "-"} - - + diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.test.tsx index 793e4c6e3cf..63c606eb00e 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.test.tsx @@ -31,12 +31,6 @@ vi.mock("antd", async () => { return { Table, Progress }; }); -vi.mock("@/utils/dataUtils", () => ({ - formatNumberWithCommas: (value: number, decimals?: number) => { - return value.toFixed(decimals || 0); - }, -})); - describe("EndpointUsageTable", () => { it("should render", () => { const mockEndpointData = { diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.tsx index f5cfb553370..88f92360ab5 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EndpointUsage/components/EndpointUsageTable.tsx @@ -1,7 +1,7 @@ import React from "react"; import { Table, Progress } from "antd"; import type { ColumnsType } from "antd/es/table"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { MoneyCell } from "@/components/shared/table_cells"; import { MetricWithMetadata } from "../../../types"; interface EndpointUsageTableProps { @@ -112,7 +112,7 @@ const EndpointUsageTable: React.FC = ({ endpointData }) title: "Spend", dataIndex: "spend", key: "spend", - render: (value: number) => `$${formatNumberWithCommas(value, 2)}`, + render: (value: number) => , }, ]; diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx index df5b14a57d6..9dd72f44f67 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/EntityUsage.tsx @@ -1,4 +1,5 @@ import useTeams from "@/app/(dashboard)/hooks/useTeams"; +import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { BarChart, @@ -665,7 +666,9 @@ const EntityUsage: React.FC = ({ accessToken, entityType, enti .map((entity) => ( {entity.metadata.alias} - ${formatNumberWithCommas(entity.metrics.spend, 4)} + + + {entity.metrics.successful_requests.toLocaleString()} @@ -777,7 +780,9 @@ const EntityUsage: React.FC = ({ accessToken, entityType, enti {provider.provider}
- ${formatNumberWithCommas(provider.spend, 2)} + + + {provider.successful_requests.toLocaleString()} diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.test.tsx index 0541a9c6925..7eea7653ef1 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.test.tsx @@ -1,4 +1,4 @@ -import { render, screen } from "@testing-library/react"; +import { fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi, beforeEach } from "vitest"; import SpendByProvider from "./SpendByProvider"; @@ -200,6 +200,14 @@ describe("SpendByProvider", () => { expect(screen.getByText("1,234,567")).toBeInTheDocument(); }); + it("should render zero spend as a dash when Show Zero Spend is on", () => { + render(); + fireEvent.click(screen.getAllByRole("switch")[0]); + expect(screen.getAllByText("google").length).toBeGreaterThan(0); + expect(screen.getByText("-")).toBeInTheDocument(); + expect(screen.queryByText("$0.00")).not.toBeInTheDocument(); + }); + it("should filter data correctly when both toggles are off", () => { render(); expect(screen.getAllByText("openai").length).toBeGreaterThan(0); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.tsx index 58d673bb7ad..5cea84affb9 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/SpendByProvider.tsx @@ -1,3 +1,4 @@ +import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { InfoCircleOutlined } from "@ant-design/icons"; import { @@ -109,7 +110,9 @@ const SpendByProvider: React.FC = ({ loading, isDateChangi {provider.provider} - ${formatNumberWithCommas(provider.spend, 2)} + + + {provider.successful_requests.toLocaleString()} {provider.failed_requests.toLocaleString()} {provider.tokens.toLocaleString()} diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.test.tsx index 126bf51bd36..766f027434f 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.test.tsx @@ -171,7 +171,9 @@ describe("TopKeyView", () => { ]} />, ); - expect(screen.getByText(/sk-1234\.\.\./)).toBeInTheDocument(); + const keyId = screen.getByText("sk-1234567890abcdef"); + expect(keyId).toBeInTheDocument(); + expect(keyId).toHaveClass("truncate"); }); it("should display dash for missing key alias", () => { @@ -206,7 +208,7 @@ describe("TopKeyView", () => { expect(screen.getByText("$123.46")).toBeInTheDocument(); }); - it("should display less than 0.01 spend as <$0.01", () => { + it("should display sub-cent spend as < $0.01", () => { render( { { api_key: "key-123", key_alias: "Test Key", - spend: 0.005, + spend: 0.004, }, ]} />, ); - expect(screen.getByText("<$0.01")).toBeInTheDocument(); + expect(screen.getByText("< $0.01")).toBeInTheDocument(); }); - it("should display zero spend correctly", () => { + it("should display zero spend as a dash", () => { render( { ]} />, ); - expect(screen.getByText("$0.00")).toBeInTheDocument(); + expect(screen.getByText("-")).toBeInTheDocument(); + expect(screen.queryByText("$0.00")).not.toBeInTheDocument(); }); it("should display dash for empty tags", () => { @@ -376,7 +379,7 @@ describe("TopKeyView", () => { />, ); - const keyIdButton = screen.getByText(/key-123\.\.\./).closest("button"); + const keyIdButton = screen.getByText("key-123").closest("button"); if (keyIdButton) { await user.click(keyIdButton); } @@ -410,7 +413,7 @@ describe("TopKeyView", () => { />, ); - const keyIdButton = screen.getByText(/key-123\.\.\./).closest("button"); + const keyIdButton = screen.getByText("key-123").closest("button"); if (keyIdButton) { await user.click(keyIdButton); } @@ -447,7 +450,7 @@ describe("TopKeyView", () => { />, ); - const keyIdButton = screen.getByText(/key-123\.\.\./).closest("button"); + const keyIdButton = screen.getByText("key-123").closest("button"); if (keyIdButton) { await user.click(keyIdButton); } @@ -483,7 +486,7 @@ describe("TopKeyView", () => { />, ); - const keyIdButton = screen.getByText(/key-123\.\.\./).closest("button"); + const keyIdButton = screen.getByText("key-123").closest("button"); if (keyIdButton) { await user.click(keyIdButton); } @@ -522,7 +525,7 @@ describe("TopKeyView", () => { />, ); - const keyIdButton = screen.getByText(/key-123\.\.\./).closest("button"); + const keyIdButton = screen.getByText("key-123").closest("button"); if (keyIdButton) { await user.click(keyIdButton); } @@ -552,7 +555,7 @@ describe("TopKeyView", () => { />, ); - const keyIdButton = screen.getByText(/key-123\.\.\./).closest("button"); + const keyIdButton = screen.getByText("key-123").closest("button"); if (keyIdButton) { await user.click(keyIdButton); } diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx index 40bc41b3e8c..2dcf98a1e5c 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopKeyView.tsx @@ -1,6 +1,7 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { IdCell, MoneyCell } from "@/components/shared/table_cells"; import { ChevronDownIcon, ChevronUpIcon } from "@heroicons/react/outline"; -import { BarChart, Button } from "@tremor/react"; +import { BarChart } from "@tremor/react"; import { Segmented, Tooltip } from "antd"; import React, { useState } from "react"; import { formatNumberWithCommas } from "../../../../utils/dataUtils"; @@ -83,20 +84,7 @@ const TopKeyView: React.FC = ({ topKeys, teams, showTags = fals { header: "Key ID", accessorKey: "api_key", - cell: (info: any) => ( -
- - - -
- ), + cell: (info: any) => handleKeyClick(info.row.original)} />, }, { header: "Key Alias", @@ -165,10 +153,7 @@ const TopKeyView: React.FC = ({ topKeys, teams, showTags = fals header: "Spend (USD)", accessorKey: "spend", meta: { numeric: true }, - cell: (info: any) => { - const value = info.getValue(); - return value > 0 && value < 0.01 ? "<$0.01" : `$${formatNumberWithCommas(value, 2)}`; - }, + cell: (info: any) => , }; const columns = showTags ? [...baseColumns, tagsColumn, spendColumn] : [...baseColumns, spendColumn]; diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.test.tsx index f6014d025ae..bbf9379f5e1 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.test.tsx @@ -175,7 +175,7 @@ describe("TopModelView", () => { setTopModelsLimit={mockSetTopModelsLimit} />, ); - expect(screen.getByText("$0.00")).toBeInTheDocument(); + expect(screen.getByText("-")).toBeInTheDocument(); expect(screen.getAllByText("0").length).toBeGreaterThan(0); }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx index 7562ef06a03..8938767c02c 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/EntityUsage/TopModelView.tsx @@ -1,6 +1,7 @@ import { BarChart } from "@tremor/react"; import { Segmented } from "antd"; import { useState } from "react"; +import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "../../../../utils/dataUtils"; import { DataTable } from "../../../view_logs/table"; @@ -31,10 +32,7 @@ export default function TopModelView({ topModels, topModelsLimit, setTopModelsLi header: "Spend (USD)", accessorKey: "spend", meta: { numeric: true }, - cell: (info: any) => { - const value = info.getValue(); - return `$${formatNumberWithCommas(value, 2)}`; - }, + cell: (info: any) => , }, { header: "Successful", diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.test.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.test.tsx index 61968294e18..322f00a501a 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.test.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.test.tsx @@ -158,7 +158,7 @@ describe("KeyModelUsageView", () => { }, ]; render(); - expect(screen.getByText("$0.00")).toBeInTheDocument(); + expect(screen.getByText("-")).toBeInTheDocument(); expect(screen.getAllByText("0").length).toBeGreaterThan(0); }); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx index ee1a49051da..ceb00e8a19f 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx @@ -1,3 +1,4 @@ +import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { BarChart, Card, Title } from "@tremor/react"; import { Table } from "antd"; @@ -24,7 +25,7 @@ const columns: ColumnsType = [ title: "Spend (USD)", dataIndex: "spend", key: "spend", - render: (value) => `$${formatNumberWithCommas(value, 2)}`, + render: (value) => , }, { title: "Successful", diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index ba616a03fd9..02f5d588149 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -1,4 +1,5 @@ -import { act, screen, waitFor, within, fireEvent } from "@testing-library/react"; +import { screen, waitFor, within, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { vi, it, expect, beforeEach, describe, MockedFunction } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; import { VirtualKeysTable } from "./VirtualKeysTable"; @@ -175,7 +176,7 @@ it("should display key information correctly", async () => { await waitFor(() => { expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); expect(screen.getByText("Test Team")).toBeInTheDocument(); - expect(screen.getByText("5.5000")).toBeInTheDocument(); + expect(screen.getByText("$5.5000")).toBeInTheDocument(); }); }); @@ -477,9 +478,8 @@ describe("Status column reflects key.blocked / scim_blocked metadata", () => { const tag = await screen.findByTestId(`key-status-${mockKey.token_id}`); expect(tag).toHaveTextContent("Blocked"); - act(() => { - fireEvent.mouseEnter(tag); - }); + const user = userEvent.setup(); + await user.hover(tag); await waitFor(() => { expect(screen.getByText(/Blocked by SCIM/i)).toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index a803d78ed57..00f4304c8a9 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -13,20 +13,10 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { - Badge, - Button, - Icon, - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Text, -} from "@tremor/react"; +import { Badge, Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Text } from "@tremor/react"; import { InfoCircleOutlined, SyncOutlined } from "@ant-design/icons"; -import { Button as AntButton, Popover, Skeleton, Tag, Tooltip, Typography } from "antd"; +import { Button as AntButton, Popover, Skeleton, Typography } from "antd"; +import { DateCell, IdCell, MoneyCell, StatusBadge } from "@/components/shared/table_cells"; import React, { useDeferredValue, useMemo, useState } from "react"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import { PaginatedKeyAliasSelect } from "../KeyAliasSelect/PaginatedKeyAliasSelect/PaginatedKeyAliasSelect"; @@ -145,23 +135,7 @@ export function VirtualKeysTable() { header: "Key ID", size: 100, enableSorting: true, - cell: (info) => { - const value = info.getValue() as string; - const width = info.cell.column.getSize(); - return ( - - - - ); - }, + cell: (info) => setSelectedKey(info.row.original)} />, }, { id: "key_alias", @@ -187,22 +161,14 @@ export function VirtualKeysTable() { cell: ({ row }) => { const key = row.original; if (key.blocked !== true) { - return ( - - Active - - ); + return ; } const isScimBlocked = (key.metadata as Record | null | undefined)?.scim_blocked === true; const reason = isScimBlocked ? "Blocked by SCIM (external identity provider deactivated or deleted the owning user)." : "Blocked. Requests using this key will be rejected with 401."; return ( - - - Blocked - - + ); }, }, @@ -323,10 +289,7 @@ export function VirtualKeysTable() { header: "Created At", size: 120, enableSorting: true, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleDateString() : "-"; - }, + cell: (info) => , }, { id: "created_by", @@ -394,10 +357,7 @@ export function VirtualKeysTable() { header: "Updated At", size: 120, enableSorting: true, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleDateString() : "Never"; - }, + cell: (info) => , }, { id: "last_active", @@ -415,16 +375,7 @@ export function VirtualKeysTable() { ), size: 130, enableSorting: false, - cell: (info) => { - const value = info.getValue(); - if (!value) return "Unknown"; - const date = new Date(value as string); - return ( - - {date.toLocaleDateString()} - - ); - }, + cell: (info) => , }, { id: "expires", @@ -432,10 +383,7 @@ export function VirtualKeysTable() { header: "Expires", size: 120, enableSorting: false, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleDateString() : "Never"; - }, + cell: (info) => , }, { id: "spend", @@ -443,7 +391,7 @@ export function VirtualKeysTable() { header: "Spend (USD)", size: 100, enableSorting: true, - cell: (info) => formatNumberWithCommas(info.getValue() as number, 4), + cell: (info) => , }, { id: "max_budget", @@ -470,10 +418,7 @@ export function VirtualKeysTable() { header: "Budget Reset", size: 130, enableSorting: false, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleString() : "Never"; - }, + cell: (info) => , }, { id: "models", diff --git a/ui/litellm-dashboard/src/components/agents.tsx b/ui/litellm-dashboard/src/components/agents.tsx index 0d8916942da..e3703f6c588 100644 --- a/ui/litellm-dashboard/src/components/agents.tsx +++ b/ui/litellm-dashboard/src/components/agents.tsx @@ -20,7 +20,7 @@ import AgentInfoView from "./agents/agent_info"; import NotificationsManager from "./molecules/notifications_manager"; import { Agent } from "./agents/types"; import { Team } from "./key_team_helpers/key_list"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { DateCell, IdCell, MoneyCell, StatusBadge } from "@/components/shared/table_cells"; import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface AgentsPanelProps { @@ -193,19 +193,10 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams {agent.agent_name} - - - + setSelectedAgentId(id)} /> - {formatNumberWithCommas(agent.spend, 4)} + @@ -213,13 +204,13 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams - {agent.created_at ? new Date(agent.created_at).toLocaleDateString() : "N/A"} + {(agent.keys?.length ?? 0) > 0 ? ( - Active + ) : ( - Needs Setup + )} {isAdmin && ( diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/plugin_table.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/plugin_table.tsx index 1646161891f..0932e658899 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/plugin_table.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/plugin_table.tsx @@ -11,6 +11,7 @@ import { import { Badge, Button, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow } from "@tremor/react"; import { Tooltip } from "antd"; import React, { useState } from "react"; +import { DateCell, IdCell, StatusBadge } from "@/components/shared/table_cells"; import NotificationsManager from "../molecules/notifications_manager"; import { getCategoryBadgeColor } from "./helpers"; import { Plugin } from "./types"; @@ -34,12 +35,6 @@ const PluginTable: React.FC = ({ }) => { const [sorting, setSorting] = useState([{ id: "created_at", desc: true }]); - const formatDate = (dateString?: string) => { - if (!dateString) return "-"; - const date = new Date(dateString); - return date.toLocaleString(); - }; - const copyToClipboard = (text: string) => { navigator.clipboard.writeText(text); NotificationsManager.success("Copied to clipboard!"); @@ -51,19 +46,9 @@ const PluginTable: React.FC = ({ accessorKey: "name", cell: ({ row }) => { const plugin = row.original; - const name = plugin.name || ""; return (
- - - + onPluginClick(plugin.id)} /> { @@ -122,24 +107,13 @@ const PluginTable: React.FC = ({ accessorKey: "enabled", cell: ({ row }) => { const plugin = row.original; - return ( - - {plugin.enabled ? "Yes" : "No"} - - ); + return ; }, }, { header: "Created At", accessorKey: "created_at", - cell: ({ row }) => { - const plugin = row.original; - return ( - - {formatDate(plugin.created_at)} - - ); - }, + cell: ({ row }) => , }, ...(isAdmin ? [ diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index 5b8dec39505..038547c6e0e 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -4,7 +4,6 @@ import { Table, TableHead, TableRow, - Badge, TableHeaderCell, TableCell, TableBody, @@ -16,7 +15,8 @@ import { import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; import { getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting } from "./networking"; import { InputNumber } from "antd"; -import { TrashIcon, CheckCircleIcon } from "@heroicons/react/outline"; +import { TrashIcon } from "@heroicons/react/outline"; +import { StatusBadge } from "@/components/shared/table_cells"; import RouterSettings from "./router_settings"; import Fallbacks from "./Settings/RouterSettings/Fallbacks/Fallbacks"; @@ -173,13 +173,11 @@ const GeneralSettings: React.FC = ({ accessToken, user {value.stored_in_db == true ? ( - - In DB - + ) : value.stored_in_db == false ? ( - In Config + ) : ( - Not Set + )} diff --git a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx index ecf6ce48fde..99f6b2793fd 100644 --- a/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx +++ b/ui/litellm-dashboard/src/components/guardrails/guardrail_table.tsx @@ -1,8 +1,8 @@ import React, { useState } from "react"; -import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon, Button } from "@tremor/react"; +import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon } from "@tremor/react"; import { TrashIcon, SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; import { Tooltip } from "antd"; -import { Badge } from "@tremor/react"; +import { DateCell, IdCell, StatusBadge } from "@/components/shared/table_cells"; import { ColumnDef, flexRender, @@ -43,13 +43,6 @@ const GuardrailTable: React.FC = ({ const [editModalVisible, setEditModalVisible] = useState(false); const [selectedGuardrail, setSelectedGuardrail] = useState(null); - // Format date helper function - const formatDate = (dateString?: string) => { - if (!dateString) return "-"; - const date = new Date(dateString); - return date.toLocaleString(); - }; - const handleEditClick = (guardrail: Guardrail) => { setSelectedGuardrail(guardrail); setEditModalVisible(true); @@ -65,18 +58,7 @@ const GuardrailTable: React.FC = ({ { header: "Guardrail ID", accessorKey: "guardrail_id", - cell: (info: any) => ( - - - - ), + cell: (info: any) => , }, { header: "Name", @@ -126,41 +108,21 @@ const GuardrailTable: React.FC = ({ header: "Default On", accessorKey: "litellm_params.default_on", cell: ({ row }) => { - const guardrail = row.original; + const isDefaultOn = !!row.original.litellm_params?.default_on; return ( - - {guardrail.litellm_params?.default_on ? "Default On" : "Default Off"} - + ); }, }, { header: "Created At", accessorKey: "created_at", - cell: ({ row }) => { - const guardrail = row.original; - return ( - - {formatDate(guardrail.created_at)} - - ); - }, + cell: ({ row }) => , }, { header: "Updated At", accessorKey: "updated_at", - cell: ({ row }) => { - const guardrail = row.original; - return ( - - {formatDate(guardrail.updated_at)} - - ); - }, + cell: ({ row }) => , }, { id: "actions", diff --git a/ui/litellm-dashboard/src/components/mcp_hub_table_columns.tsx b/ui/litellm-dashboard/src/components/mcp_hub_table_columns.tsx index 7cf0d48a49f..1e25f87d262 100644 --- a/ui/litellm-dashboard/src/components/mcp_hub_table_columns.tsx +++ b/ui/litellm-dashboard/src/components/mcp_hub_table_columns.tsx @@ -2,6 +2,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { Button, Badge, Text } from "@tremor/react"; import { Tooltip, Tag } from "antd"; import { CopyOutlined, InfoCircleOutlined } from "@ant-design/icons"; +import { StatusBadge, type StatusTone } from "@/components/shared/table_cells"; export interface MCPServerData { server_id: string; @@ -124,21 +125,17 @@ export const mcpHubColumns = ( cell: ({ row }) => { const server = row.original; - const statusColors: Record = { - active: "green", - inactive: "red", - unknown: "gray", - healthy: "green", - unhealthy: "red", + const statusTones: Record = { + active: "success", + inactive: "error", + unknown: "neutral", + healthy: "success", + unhealthy: "error", }; - const color = statusColors[server.status] || "gray"; + const tone = statusTones[server.status] || "neutral"; - return ( - - {server.status || "unknown"} - - ); + return ; }, }, { diff --git a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx index 546df9ebc4b..5e7e99ee8ec 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/MCPToolsetsTab.tsx @@ -6,6 +6,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets"; import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"; import { useQueryClient } from "@tanstack/react-query"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { DataTable } from "../view_logs/table"; import { createMCPToolset, updateMCPToolset, deleteMCPToolset, listMCPTools, getProxyBaseUrl } from "../networking"; import { MCPToolset, MCPToolsetTool } from "./types"; @@ -302,11 +303,7 @@ function toolsetColumns( { header: "Toolset ID", accessorKey: "toolset_id", - cell: ({ row }) => ( - - {row.original.toolset_id.slice(0, 8)}… - - ), + cell: ({ row }) => , }, { header: "Name", @@ -359,11 +356,7 @@ function toolsetColumns( { header: "Created", accessorKey: "created_at", - cell: ({ row }) => ( - - {row.original.created_at ? new Date(row.original.created_at).toLocaleDateString() : "—"} - - ), + cell: ({ row }) => , }, ...(isAdmin ? [ diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index a2194e23b6e..6497f2686cb 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect, useRef } from "react"; -import { Title, Text, Button, Badge } from "@tremor/react"; +import { Title, Text, Button } from "@tremor/react"; import { Modal } from "antd"; import { Button as AntdButton } from "antd"; import { ModelDataTable } from "./table"; @@ -468,21 +468,6 @@ const HealthCheckComponent: React.FC = ({ onPageChange?.(page); }; - const getStatusBadge = (status: string) => { - switch (status) { - case "healthy": - return healthy; - case "unhealthy": - return unhealthy; - case "checking": - return checking; - case "none": - return none; - default: - return unknown; - } - }; - const showErrorModal = (modelName: string, cleanedError: string, fullError: string) => { setSelectedErrorDetails({ modelName, @@ -612,7 +597,6 @@ const HealthCheckComponent: React.FC = ({ handleModelSelection, handleSelectAll, runIndividualHealthCheck, - getStatusBadge, getDisplayModelName, showErrorModal, showSuccessModal, diff --git a/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx b/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx index 12cc984ef91..33c97236f97 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/health_check_columns.tsx @@ -3,6 +3,7 @@ import { Tooltip, Checkbox } from "antd"; import { Text } from "@tremor/react"; import { InformationCircleIcon, PlayIcon, RefreshIcon } from "@heroicons/react/outline"; import { Team } from "@/components/key_team_helpers/key_list"; +import { IdCell, StatusBadge, type StatusTone } from "@/components/shared/table_cells"; interface HealthCheckData { model_name: string; @@ -21,6 +22,18 @@ interface HealthCheckData { health_full_error?: string; } +const HEALTH_STATUS_TONES: Record = { + healthy: "success", + unhealthy: "error", + checking: "info", + none: "neutral", +}; + +const healthStatusBadge = (status: string): JSX.Element => { + const tone = HEALTH_STATUS_TONES[status]; + return tone ? : ; +}; + interface HealthStatus { status: string; lastCheck: string; @@ -38,7 +51,6 @@ export const healthCheckColumns = ( handleModelSelection: (modelId: string, checked: boolean) => void, handleSelectAll: (checked: boolean) => void, runIndividualHealthCheck: (modelId: string) => void, - getStatusBadge: (status: string) => JSX.Element, getDisplayModelName: (model: any) => string, showErrorModal?: (modelName: string, cleanedError: string, fullError: string) => void, showSuccessModal?: (modelName: string, response: any) => void, @@ -72,14 +84,7 @@ export const healthCheckColumns = ( onChange={(e) => handleModelSelection(modelId, e.target.checked)} onClick={(e) => e.stopPropagation()} /> - -
setSelectedModelId && setSelectedModelId(model.model_info.id)} - > - {model.model_info.id} -
- +
); }, @@ -175,7 +180,7 @@ export const healthCheckColumns = ( return (
- {getStatusBadge(healthStatus.status)} + {healthStatusBadge(healthStatus.status)} {hasSuccessResponse && showSuccessModal && ( - +
e.stopPropagation()}> +
) : ( "-" @@ -370,15 +345,10 @@ export const columns = ( minSize: 80, cell: ({ row }) => { const model = row.original; - return ( -
- {model.model_info.db_model ? "DB Model" : "Config Model"} -
+ return model.model_info.db_model ? ( + + ) : ( + ); }, }, diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.tsx index e1b8c6d5044..402cc33902f 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.tsx @@ -1,6 +1,7 @@ import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { organizationKeys, useOrganization } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; import { useQueryClient } from "@tanstack/react-query"; +import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas, copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils"; import { createTeamAliasMap } from "@/utils/teamUtils"; import { ArrowLeftIcon } from "@heroicons/react/outline"; @@ -196,7 +197,7 @@ const OrganizationInfoView: React.FC = ({ render: (_: unknown, record: Member) => { const orgMember = record.user_id != null ? (orgData.members || []).find((m) => m.user_id === record.user_id) : undefined; - return ${formatNumberWithCommas(orgMember?.spend ?? 0, 4)}; + return ; }, }, { diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index 857a0a0c64c..edebc17087a 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -27,7 +27,7 @@ import { import { Form, Input, Modal, Select as Select2, Tooltip } from "antd"; import { useQueryClient } from "@tanstack/react-query"; import React, { useState } from "react"; -import { formatNumberWithCommas } from "../utils/dataUtils"; +import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; @@ -263,30 +263,22 @@ const OrganizationsTable: React.FC = ({ .map((org: Organization) => ( -
- - - -
+ {org.organization_alias} - {org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"} + - {formatNumberWithCommas(org.spend, 4)} - {org.litellm_budget_table?.max_budget !== null && - org.litellm_budget_table?.max_budget !== undefined - ? org.litellm_budget_table?.max_budget - : "No limit"} + + + + = ({ { header: "ID", accessorKey: "id", - cell: (info: any) => ( - -
info.row.original.id && setSelectedEndpointId(info.row.original.id)} - > - {info.row.original.id} -
- - ), + cell: (info: any) => , }, { header: "Path", @@ -192,7 +184,9 @@ const PassThroughSettings: React.FC = ({
), accessorKey: "auth", - cell: (info: any) => {info.getValue() ? "Yes" : "No"}, + cell: (info: any) => ( + + ), }, { header: "Headers", diff --git a/ui/litellm-dashboard/src/components/policies/attachment_table.test.tsx b/ui/litellm-dashboard/src/components/policies/attachment_table.test.tsx index 099aa97a433..0cfd4e41e0a 100644 --- a/ui/litellm-dashboard/src/components/policies/attachment_table.test.tsx +++ b/ui/litellm-dashboard/src/components/policies/attachment_table.test.tsx @@ -140,10 +140,13 @@ describe("AttachmentTable", () => { expect(screen.queryByRole("button", { name: /TrashIcon/i })).not.toBeInTheDocument(); }); - it("should show a truncated attachment ID in the table", () => { + it("should show the attachment ID as truncated plain mono text", () => { const attachment = makeAttachment({ attachment_id: "att-abcdef1234567" }); renderWithProviders(); - expect(screen.getByText("att-abc...")).toBeInTheDocument(); + const idElement = screen.getByText("att-abcdef1234567"); + expect(idElement.className).toContain("font-mono"); + expect(idElement.className).toContain("truncate"); + expect(idElement.className).not.toContain("bg-blue-50"); }); it("should render model tags when the attachment has models", () => { diff --git a/ui/litellm-dashboard/src/components/policies/attachment_table.tsx b/ui/litellm-dashboard/src/components/policies/attachment_table.tsx index d9de8378a8a..fa482552fd5 100644 --- a/ui/litellm-dashboard/src/components/policies/attachment_table.tsx +++ b/ui/litellm-dashboard/src/components/policies/attachment_table.tsx @@ -10,6 +10,7 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { PolicyAttachment } from "./types"; import ImpactPopover from "./impact_popover"; @@ -30,24 +31,11 @@ const AttachmentTable: React.FC = ({ }) => { const [sorting, setSorting] = useState([{ id: "created_at", desc: true }]); - // Format date helper function - const formatDate = (dateString?: string) => { - if (!dateString) return "-"; - const date = new Date(dateString); - return date.toLocaleString(); - }; - const columns: ColumnDef[] = [ { header: "Attachment ID", accessorKey: "attachment_id", - cell: (info: any) => ( - - - {info.getValue() ? `${String(info.getValue()).slice(0, 7)}...` : ""} - - - ), + cell: (info: any) => , }, { header: "Policy", @@ -183,14 +171,7 @@ const AttachmentTable: React.FC = ({ { header: "Created At", accessorKey: "created_at", - cell: ({ row }) => { - const attachment = row.original; - return ( - - {formatDate(attachment.created_at)} - - ); - }, + cell: ({ row }) => , }, { id: "actions", diff --git a/ui/litellm-dashboard/src/components/policies/policy_table.tsx b/ui/litellm-dashboard/src/components/policies/policy_table.tsx index eafec442474..1716f5aea84 100644 --- a/ui/litellm-dashboard/src/components/policies/policy_table.tsx +++ b/ui/litellm-dashboard/src/components/policies/policy_table.tsx @@ -10,6 +10,7 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; +import { DateCell } from "@/components/shared/table_cells"; import { Policy } from "./types"; /** One row per policy name; primaryPolicy is used for display and for Edit (FlowBuilder loads all versions) */ @@ -59,12 +60,6 @@ const PolicyTable: React.FC = ({ const rows = useMemo(() => groupPoliciesByName(policies), [policies]); - const formatDate = (dateString?: string) => { - if (!dateString) return "-"; - const date = new Date(dateString); - return date.toLocaleString(); - }; - const columns: ColumnDef[] = [ { header: "Name", @@ -199,14 +194,7 @@ const PolicyTable: React.FC = ({ header: "Created At", id: "created_at", accessorFn: (row) => row.primaryPolicy.created_at ?? "", - cell: ({ row }) => { - const policy = row.original.primaryPolicy; - return ( - - {formatDate(policy.created_at)} - - ); - }, + cell: ({ row }) => , }, { id: "actions", diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/cell_tooltip.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/cell_tooltip.tsx new file mode 100644 index 00000000000..c6e10590e8f --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/cell_tooltip.tsx @@ -0,0 +1,21 @@ +"use client"; + +import * as React from "react"; + +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; + +interface CellTooltipProps { + content: React.ReactNode; + trigger: React.ReactElement; +} + +export function CellTooltip({ content, trigger }: CellTooltipProps) { + return ( + + + + {content} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/date_cell.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/date_cell.test.tsx new file mode 100644 index 00000000000..719047e0cfe --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/date_cell.test.tsx @@ -0,0 +1,57 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { DateCell, formatCellDate, formatFullTimestamp } from "./date_cell"; + +const localIso = new Date(2026, 6, 7, 9, 50, 13).toISOString(); + +describe("formatCellDate", () => { + it("formats datetime precision as 'MMM D, HH:mm:ss' without a year", () => { + expect(formatCellDate(new Date(2026, 6, 7, 9, 50, 13), "datetime")).toBe("Jul 7, 09:50:13"); + }); + + it("zero-pads hours, minutes and seconds", () => { + expect(formatCellDate(new Date(2026, 0, 2, 1, 2, 3), "datetime")).toBe("Jan 2, 01:02:03"); + }); + + it("formats date precision as 'MMM D, YYYY' with no time", () => { + expect(formatCellDate(new Date(2026, 11, 31, 23, 59, 59), "date")).toBe("Dec 31, 2026"); + }); +}); + +describe("formatFullTimestamp", () => { + it("includes year, 24h time and the IANA timezone", () => { + const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; + expect(formatFullTimestamp(new Date(2026, 6, 7, 9, 50, 13))).toBe(`Jul 7, 2026, 09:50:13 (${timeZone})`); + }); +}); + +describe("DateCell", () => { + it("renders the datetime format by default", () => { + render(); + expect(screen.getByText("Jul 7, 09:50:13")).toBeInTheDocument(); + }); + + it("renders date-only when precision is 'date'", () => { + render(); + expect(screen.getByText("Jul 7, 2026")).toBeInTheDocument(); + }); + + it("renders '-' for null and undefined", () => { + const { rerender } = render(); + expect(screen.getByText("-")).toBeInTheDocument(); + rerender(); + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("renders the custom fallback for empty values", () => { + render(); + expect(screen.getByText("Never")).toBeInTheDocument(); + }); + + it("renders the fallback instead of 'Invalid Date' for unparseable input", () => { + render(); + expect(screen.getByText("Unknown")).toBeInTheDocument(); + expect(screen.queryByText(/Invalid/)).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/date_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/date_cell.tsx new file mode 100644 index 00000000000..ee4c01bb237 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/date_cell.tsx @@ -0,0 +1,41 @@ +"use client"; + +import { CellTooltip } from "./cell_tooltip"; + +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] as const; + +export type DatePrecision = "datetime" | "date"; + +interface DateCellProps { + value: string | null | undefined; + precision?: DatePrecision; + fallback?: string; +} + +const pad = (n: number): string => String(n).padStart(2, "0"); + +export const formatCellDate = (date: Date, precision: DatePrecision): string => + precision === "date" + ? `${MONTHS[date.getMonth()]} ${date.getDate()}, ${date.getFullYear()}` + : `${MONTHS[date.getMonth()]} ${date.getDate()}, ${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`; + +export const formatFullTimestamp = (date: Date): string => { + const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; + const day = `${MONTHS[date.getMonth()]} ${date.getDate()}, ${date.getFullYear()}`; + const time = `${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`; + return `${day}, ${time} (${timeZone})`; +}; + +export function DateCell({ value, precision = "datetime", fallback = "-" }: DateCellProps) { + const date = value ? new Date(value) : null; + if (!date || Number.isNaN(date.getTime())) { + return {fallback}; + } + + return ( + {formatCellDate(date, precision)}} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx new file mode 100644 index 00000000000..1a87f17d50b --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.test.tsx @@ -0,0 +1,78 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import { IdCell } from "./id_cell"; + +const { copyToClipboardMock } = vi.hoisted(() => ({ copyToClipboardMock: vi.fn() })); + +vi.mock("@/utils/dataUtils", async (importOriginal) => ({ + ...(await importOriginal()), + copyToClipboard: copyToClipboardMock, +})); + +describe("IdCell", () => { + it("renders '-' for empty values", () => { + render(); + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("renders the custom fallback for empty values", () => { + render(); + expect(screen.getByText("—")).toBeInTheDocument(); + }); + + it("renders the full id as a non-interactive pill by default", () => { + render(); + const el = screen.getByText("sk-1234567890abcdef"); + expect(el.tagName).toBe("SPAN"); + expect(el.className).toContain("bg-blue-50"); + expect(el.className).toContain("font-mono"); + expect(el.className).toContain("max-w-[15ch]"); + expect(el.className).toContain("truncate"); + }); + + it("renders plain mono text without pill styling for the plain variant", () => { + render(); + const el = screen.getByText("req-123"); + expect(el.className).toContain("font-mono"); + expect(el.className).not.toContain("bg-blue-50"); + }); + + it("does not truncate when truncate is false", () => { + render(); + expect(screen.getByText("audit-object-id").className).not.toContain("truncate"); + }); + + it("becomes a button that fires onClick with the id value", async () => { + const user = userEvent.setup(); + const onClick = vi.fn(); + render(); + await user.click(screen.getByRole("button", { name: "team-42" })); + expect(onClick).toHaveBeenCalledWith("team-42"); + }); + + it("stays non-interactive when disabled, even with onClick", () => { + const onClick = vi.fn(); + render(); + expect(screen.queryByRole("button", { name: "tag-1" })).not.toBeInTheDocument(); + }); + + it("copies the id via the trailing copy button without triggering row clicks", async () => { + const user = userEvent.setup(); + const rowClick = vi.fn(); + render( +
+ +
, + ); + await user.click(screen.getByRole("button", { name: "Copy ID" })); + expect(copyToClipboardMock).toHaveBeenCalledWith("key-hash-9"); + expect(rowClick).not.toHaveBeenCalled(); + }); + + it("passes dataTestId through to the id element", () => { + render(); + expect(screen.getByTestId("key-id-cell")).toHaveTextContent("k-1"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx new file mode 100644 index 00000000000..6fbd2e2f9ed --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/id_cell.tsx @@ -0,0 +1,94 @@ +"use client"; + +import { Copy } from "lucide-react"; +import * as React from "react"; + +import { cn } from "@/lib/cva.config"; +import { copyToClipboard } from "@/utils/dataUtils"; + +import { CellTooltip } from "./cell_tooltip"; + +export type IdCellVariant = "pill" | "plain"; + +interface IdCellProps { + value: string | null | undefined; + variant?: IdCellVariant; + onClick?: (value: string) => void; + copyable?: boolean; + truncate?: boolean; + fallback?: string; + tooltip?: React.ReactNode; + disabled?: boolean; + dataTestId?: string; + className?: string; +} + +const VARIANT_CLASS: Record = { + pill: { + base: "font-mono text-xs font-normal px-2 py-0.5 rounded-md text-left bg-blue-50 text-blue-500", + clickable: "hover:bg-blue-100 cursor-pointer", + }, + plain: { + base: "font-mono text-xs text-left", + clickable: "hover:text-blue-600 cursor-pointer", + }, +}; + +export function IdCell({ + value, + variant = "pill", + onClick, + copyable = false, + truncate = true, + fallback = "-", + tooltip, + disabled = false, + dataTestId, + className, +}: IdCellProps) { + if (!value) { + return {fallback}; + } + + const clickable = !!onClick && !disabled; + const classes = cn( + VARIANT_CLASS[variant].base, + clickable && VARIANT_CLASS[variant].clickable, + truncate && "block max-w-[15ch] truncate", + disabled && "opacity-50", + className, + ); + + const idElement = clickable ? ( + + ) : ( + + {value} + + ); + + const withTooltip = ; + + if (!copyable) { + return withTooltip; + } + + return ( + + {withTooltip} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/index.ts b/ui/litellm-dashboard/src/components/shared/table_cells/index.ts new file mode 100644 index 00000000000..e189413d43d --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/index.ts @@ -0,0 +1,5 @@ +export { CellTooltip } from "./cell_tooltip"; +export { DateCell, formatCellDate, formatFullTimestamp, type DatePrecision } from "./date_cell"; +export { IdCell, type IdCellVariant } from "./id_cell"; +export { MoneyCell } from "./money_cell"; +export { StatusBadge, type StatusTone } from "./status_badge"; diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/money_cell.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/money_cell.test.tsx new file mode 100644 index 00000000000..473785e31c4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/money_cell.test.tsx @@ -0,0 +1,44 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { MoneyCell } from "./money_cell"; + +describe("MoneyCell", () => { + it("renders '-' for null and undefined", () => { + const { rerender } = render(); + expect(screen.getByText("-")).toBeInTheDocument(); + rerender(); + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("renders the custom emptyText for null budgets", () => { + render(); + expect(screen.getByText("Unlimited")).toBeInTheDocument(); + }); + + it("renders '-' for zero by default", () => { + render(); + expect(screen.getByText("-")).toBeInTheDocument(); + }); + + it("renders a formatted zero when showZero is set, never the emptyText", () => { + render(); + expect(screen.getByText("$0.00")).toBeInTheDocument(); + expect(screen.queryByText("Unlimited")).not.toBeInTheDocument(); + }); + + it("formats amounts with commas, a dollar sign and the given decimals", () => { + render(); + expect(screen.getByText("$1,234.57")).toBeInTheDocument(); + }); + + it("defaults to 4 decimals", () => { + render(); + expect(screen.getByText("$42.0000")).toBeInTheDocument(); + }); + + it("renders the sub-threshold form for amounts that round to zero", () => { + render(); + expect(screen.getByText("< $0.000001")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/money_cell.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/money_cell.tsx new file mode 100644 index 00000000000..9d3c747b20e --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/money_cell.tsx @@ -0,0 +1,23 @@ +"use client"; + +import { formatNumberWithCommas, getSpendString } from "@/utils/dataUtils"; + +interface MoneyCellProps { + value: number | null | undefined; + decimals?: number; + emptyText?: string; + showZero?: boolean; +} + +export function MoneyCell({ value, decimals = 4, emptyText = "-", showZero = false }: MoneyCellProps) { + if (value === null || value === undefined || Number.isNaN(value)) { + return {emptyText}; + } + if (value === 0) { + if (!showZero) { + return -; + } + return {`$${formatNumberWithCommas(0, decimals, false, true)}`}; + } + return {getSpendString(value, decimals)}; +} diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/status_badge.test.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/status_badge.test.tsx new file mode 100644 index 00000000000..724a362c21f --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/status_badge.test.tsx @@ -0,0 +1,42 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; + +import { StatusBadge, type StatusTone } from "./status_badge"; + +describe("StatusBadge", () => { + const toneClasses: Record = { + success: ["border-green-200", "bg-green-50", "text-green-600"], + error: ["border-red-200", "bg-red-50", "text-red-600"], + warning: ["border-amber-200", "bg-amber-50", "text-amber-600"], + neutral: ["border-gray-200", "bg-gray-50", "text-gray-600"], + info: ["border-blue-200", "bg-blue-50", "text-blue-600"], + }; + + (Object.entries(toneClasses) as [StatusTone, string[]][]).forEach(([tone, classes]) => { + it(`renders a tinted pill (${classes.join(" ")}) for the ${tone} tone`, () => { + render(); + const badge = screen.getByText(tone); + classes.forEach((cls) => expect(badge.className).toContain(cls)); + }); + }); + + it("renders the label text inside an outline badge with no status dot", () => { + render(); + const badge = screen.getByText("Active"); + expect(badge.dataset.variant).toBe("outline"); + expect(badge.querySelector("[aria-hidden]")).toBeNull(); + }); + + it("passes dataTestId through", () => { + render(); + expect(screen.getByTestId("key-status")).toHaveTextContent("Blocked"); + }); + + it("opens the tooltip on hover, which requires Badge to forward its ref to the trigger", async () => { + const user = userEvent.setup(); + render(); + await user.hover(screen.getByText("Blocked")); + expect(await screen.findByText("This key was blocked by SCIM")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/table_cells/status_badge.tsx b/ui/litellm-dashboard/src/components/shared/table_cells/status_badge.tsx new file mode 100644 index 00000000000..f10f817650a --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/table_cells/status_badge.tsx @@ -0,0 +1,38 @@ +"use client"; + +import * as React from "react"; + +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/cva.config"; + +import { CellTooltip } from "./cell_tooltip"; + +export type StatusTone = "success" | "error" | "warning" | "neutral" | "info"; + +const TONE_CLASS: Record = { + success: "border-green-200 bg-green-50 text-green-600", + error: "border-red-200 bg-red-50 text-red-600", + warning: "border-amber-200 bg-amber-50 text-amber-600", + neutral: "border-gray-200 bg-gray-50 text-gray-600", + info: "border-blue-200 bg-blue-50 text-blue-600", +}; + +interface StatusBadgeProps { + tone: StatusTone; + label: string; + tooltip?: React.ReactNode; + dataTestId?: string; +} + +export function StatusBadge({ tone, label, tooltip, dataTestId }: StatusBadgeProps) { + const badge = ( + + {label} + + ); + + if (!tooltip) { + return badge; + } + return ; +} diff --git a/ui/litellm-dashboard/src/components/skill_hub_table_columns.tsx b/ui/litellm-dashboard/src/components/skill_hub_table_columns.tsx index 8a8d7ae7042..8fc9adc75a2 100644 --- a/ui/litellm-dashboard/src/components/skill_hub_table_columns.tsx +++ b/ui/litellm-dashboard/src/components/skill_hub_table_columns.tsx @@ -3,6 +3,7 @@ import { Badge, Text } from "@tremor/react"; import { Tooltip } from "antd"; import { CopyOutlined, LinkOutlined } from "@ant-design/icons"; import { Plugin } from "./claude_code_plugins/types"; +import { StatusBadge } from "@/components/shared/table_cells"; export const skillHubColumns = ( showModal: (skill: Plugin) => void, @@ -104,9 +105,10 @@ export const skillHubColumns = ( accessorKey: "enabled", enableSorting: true, cell: ({ row }) => ( - - {row.original.enabled ? "Public" : "Draft"} - + ), }, ]; diff --git a/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx b/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx index a56721787d5..057f30ccee5 100644 --- a/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx @@ -1,5 +1,6 @@ -import { render, screen } from "@testing-library/react"; +import { fireEvent, render, screen } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { formatCellDate } from "@/components/shared/table_cells"; import TagTable from "./TagTable"; import { Tag } from "./types"; @@ -75,14 +76,21 @@ describe("TagTable", () => { it("should display formatted created date", () => { render(); - const formattedDate = new Date(mockTag.created_at).toLocaleDateString(); + const formattedDate = formatCellDate(new Date(mockTag.created_at), "date"); expect(screen.getByText(formattedDate)).toBeInTheDocument(); }); - it("should disable tag name button for dynamic spend tags", () => { + it("should call onSelectTag when tag name is clicked", () => { + render(); + fireEvent.click(screen.getByRole("button", { name: "test-tag" })); + expect(mockOnSelectTag).toHaveBeenCalledWith("test-tag"); + }); + + it("should render tag name as non-clickable for dynamic spend tags", () => { render(); - const tagButton = screen.getByRole("button", { name: "dynamic-spend-tag" }); - expect(tagButton).toBeDisabled(); + expect(screen.queryByRole("button", { name: "dynamic-spend-tag" })).not.toBeInTheDocument(); + fireEvent.click(screen.getByText("dynamic-spend-tag")); + expect(mockOnSelectTag).not.toHaveBeenCalled(); }); it("should disable edit icon for dynamic spend tags", () => { diff --git a/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx b/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx index ce28ac6e6f2..e34653cc702 100644 --- a/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx @@ -7,20 +7,10 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { - Badge, - Button, - Icon, - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Text, -} from "@tremor/react"; +import { Badge, Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Text } from "@tremor/react"; import { Tooltip } from "antd"; import React from "react"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { Tag } from "./types"; interface TagTableProps { @@ -45,21 +35,15 @@ const TagTable: React.FC = ({ data, onEdit, onDelete, onSelectTag const isDynamicSpendTag = tag.description === DYNAMIC_SPEND_TAG_DESCRIPTION; return (
- - - + />
); }, @@ -104,10 +88,7 @@ const TagTable: React.FC = ({ data, onEdit, onDelete, onSelectTag header: "Created", accessorKey: "created_at", sortingFn: "datetime", - cell: ({ row }) => { - const tag = row.original; - return {new Date(tag.created_at).toLocaleDateString()}; - }, + cell: ({ row }) => , }, { id: "actions", diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx index ac0ae16a44f..a07c57eaa30 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.test.tsx @@ -27,6 +27,8 @@ const mockSetSelectedEditMember = vi.fn(); const mockSetIsEditMemberModalVisible = vi.fn(); const mockSetIsAddMemberModalVisible = vi.fn(); +const budgetResetIso = new Date(2026, 6, 15, 12, 0, 0).toISOString(); + const createMockTeamData = (overrides: Partial = {}): TeamData => ({ team_id: "team-123", team_info: { @@ -78,6 +80,7 @@ const createMockTeamData = (overrides: Partial = {}): TeamData => ({ rpm_limit: 100, model_max_budget: null, budget_duration: null, + budget_reset_at: budgetResetIso, }, }, ], @@ -202,7 +205,7 @@ describe("TeamMembersComponent", () => { />, ); - expect(screen.getByText("-")).toBeInTheDocument(); + expect(screen.getAllByText("-").length).toBeGreaterThanOrEqual(1); }); it("should display Default Proxy Admin tag for default_user_id", () => { @@ -243,12 +246,12 @@ describe("TeamMembersComponent", () => { />, ); - expect(screen.getByText(/\$100\.5/)).toBeInTheDocument(); + expect(screen.getByText("$100.5000")).toBeInTheDocument(); expect(screen.getByText(/100 RPM/)).toBeInTheDocument(); expect(screen.getByText(/10000 TPM/)).toBeInTheDocument(); }); - it("should display No Limit for budget when member has no budget", () => { + it("should display the budget reset date for member with a budget reset", () => { renderWithProviders( { />, ); - expect(screen.getByText("No Limit")).toBeInTheDocument(); + expect(screen.getByText("Jul 15, 2026")).toBeInTheDocument(); + }); + + it("should display formatted budget and Unlimited for member with no budget", () => { + renderWithProviders( + , + ); + + expect(screen.getByText("$1,000.0000")).toBeInTheDocument(); + expect(screen.getByText("Unlimited")).toBeInTheDocument(); }); it("should display No Limits for rate limits when member has no limits", () => { diff --git a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx index e2f108dcbf5..b884490efc0 100644 --- a/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamMemberTab.tsx @@ -1,7 +1,7 @@ import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { Member } from "@/components/networking"; -import { formatBudgetReset } from "@/utils/budgetUtils"; +import { DateCell, MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { isProxyAdminRole, isUserTeamAdminForSingleTeam } from "@/utils/roles"; import { InfoCircleOutlined } from "@ant-design/icons"; @@ -58,14 +58,10 @@ export default function TeamMemberTab({ return membership?.total_spend ?? 0; }; - const getUserBudget = (userId: string | null): string | null => { + const getUserBudget = (userId: string | null): number | null => { if (!userId) return null; const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); - const maxBudget = membership?.litellm_budget_table?.max_budget; - if (maxBudget === null || maxBudget === undefined) { - return null; - } - return formatNumber(maxBudget); + return membership?.litellm_budget_table?.max_budget ?? null; }; // Helper function to get rate limits for a user @@ -98,7 +94,7 @@ export default function TeamMemberTab({ const getUserBudgetReset = (userId: string | null): string | null => { if (!userId) return null; const membership = teamData.team_memberships.find((tm) => tm.user_id === userId); - return formatBudgetReset(membership?.litellm_budget_table?.budget_reset_at); + return membership?.litellm_budget_table?.budget_reset_at ?? null; }; const extraColumns: ColumnsType = [ @@ -146,7 +142,7 @@ export default function TeamMemberTab({ ), key: "spend", render: (_: unknown, record: Member) => ( - ${formatNumberWithCommas(getUserCurrentCycleSpend(record.user_id), 4)} + ), }, { @@ -159,31 +155,19 @@ export default function TeamMemberTab({ ), key: "total_spend", - render: (_: unknown, record: Member) => ( - ${formatNumberWithCommas(getUserTotalSpend(record.user_id), 4)} - ), + render: (_: unknown, record: Member) => , }, { title: "Team Member Budget (USD)", key: "budget", - render: (_: unknown, record: Member) => { - const budget = getUserBudget(record.user_id); - return ( - {budget ? `$${formatNumberWithCommas(Number(budget), 4)}` : "No Limit"} - ); - }, + render: (_: unknown, record: Member) => ( + + ), }, { title: "Budget Reset", key: "budget_reset", - render: (_: unknown, record: Member) => { - const reset = getUserBudgetReset(record.user_id); - return reset ? ( - {reset} - ) : ( - — - ); - }, + render: (_: unknown, record: Member) => , }, { title: ( diff --git a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx index e51c124d618..b7128e642a5 100644 --- a/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamVirtualKeysTable.tsx @@ -2,7 +2,7 @@ "use client"; import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; import { ChevronDownIcon, ChevronRightIcon, ChevronUpIcon, SwitchVerticalIcon } from "@heroicons/react/outline"; import { ColumnDef, @@ -12,18 +12,7 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { - Badge, - Button, - Icon, - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Text, -} from "@tremor/react"; +import { Badge, Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Text } from "@tremor/react"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Popover, Skeleton, Tooltip, Typography } from "antd"; import DefaultProxyAdminTag from "../common_components/DefaultProxyAdminTag"; @@ -214,23 +203,9 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Key ID", size: 100, enableSorting: true, - cell: (info) => { - const value = info.getValue() as string; - const width = info.cell.column.getSize(); - return ( - - - - ); - }, + cell: (info) => ( + setSelectedKey(info.row.original)} /> + ), }, { id: "key_alias", @@ -310,10 +285,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Created At", size: 120, enableSorting: true, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleDateString() : "-"; - }, + cell: (info) => , }, { id: "created_by", @@ -380,10 +352,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Updated At", size: 120, enableSorting: true, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleDateString() : "Never"; - }, + cell: (info) => , }, { id: "last_active", @@ -401,16 +370,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi ), size: 130, enableSorting: false, - cell: (info) => { - const value = info.getValue(); - if (!value) return "Unknown"; - const date = new Date(value as string); - return ( - - {date.toLocaleDateString()} - - ); - }, + cell: (info) => , }, { id: "expires", @@ -418,10 +378,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Expires", size: 120, enableSorting: false, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleDateString() : "Never"; - }, + cell: (info) => , }, { id: "spend", @@ -429,7 +386,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Spend (USD)", size: 100, enableSorting: true, - cell: (info) => formatNumberWithCommas(info.getValue() as number, 4), + cell: (info) => , }, { id: "max_budget", @@ -437,11 +394,9 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Budget (USD)", size: 110, enableSorting: true, - cell: (info) => { - const maxBudget = info.getValue() as number | null; - if (maxBudget === null) return "Unlimited"; - return `$${formatNumberWithCommas(maxBudget)}`; - }, + cell: (info) => ( + + ), }, { id: "budget_reset_at", @@ -449,10 +404,7 @@ export function TeamVirtualKeysTable({ teamId, teamAlias, organization }: TeamVi header: "Budget Reset", size: 130, enableSorting: false, - cell: (info) => { - const value = info.getValue(); - return value ? new Date(value as string).toLocaleString() : "Never"; - }, + cell: (info) => , }, { id: "models", diff --git a/ui/litellm-dashboard/src/components/ui/badge.tsx b/ui/litellm-dashboard/src/components/ui/badge.tsx index 87b536cc37e..2e1ebffa109 100644 --- a/ui/litellm-dashboard/src/components/ui/badge.tsx +++ b/ui/litellm-dashboard/src/components/ui/badge.tsx @@ -21,14 +21,18 @@ const badgeVariants = cva({ }, }); -function Badge({ - className, - variant = "default", - ...props -}: React.ComponentProps<"span"> & VariantProps) { - return ( - - ); -} +const Badge = React.forwardRef< + HTMLSpanElement, + React.ComponentPropsWithoutRef<"span"> & VariantProps +>(({ className, variant = "default", ...props }, ref) => ( + +)); +Badge.displayName = "Badge"; export { Badge, badgeVariants }; diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 04fcbddbd37..91c12fd1fa2 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -50,6 +50,7 @@ import { getProxyUISettings, } from "./networking"; import TopKeyView from "./UsagePage/components/EntityUsage/TopKeyView"; +import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; interface UsagePageProps { @@ -644,9 +645,7 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use {provider.provider} - {parseFloat(provider.spend.toFixed(2)) < 0.00001 - ? "less than 0.00" - : formatNumberWithCommas(provider.spend, 2)} + ))} @@ -819,7 +818,9 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use {topUsers?.map((user: any, index: number) => ( {user.end_user} - {formatNumberWithCommas(user.total_spend, 2)} + + + {user.total_count} ))} diff --git a/ui/litellm-dashboard/src/components/vector_store_management/DocumentsTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/DocumentsTable.tsx index c8288172d5b..eaa864ab3dc 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/DocumentsTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/DocumentsTable.tsx @@ -1,7 +1,8 @@ import React from "react"; -import { Table, Badge, Tooltip } from "antd"; +import { Table, Tooltip } from "antd"; import MessageManager from "@/components/molecules/message_manager"; import { EyeOutlined, CopyOutlined, DeleteOutlined } from "@ant-design/icons"; +import { StatusBadge, type StatusTone } from "@/components/shared/table_cells"; import { DocumentUpload } from "./types"; interface DocumentsTableProps { @@ -16,15 +17,15 @@ const DocumentsTable: React.FC = ({ documents, onRemove }) }; const getStatusBadge = (status: DocumentUpload["status"]) => { - const statusConfig = { - uploading: { color: "blue", text: "Uploading" }, - done: { color: "green", text: "Ready" }, - error: { color: "red", text: "Error" }, - removed: { color: "default", text: "Removed" }, + const statusConfig: Record = { + uploading: { tone: "info", label: "Uploading" }, + done: { tone: "success", label: "Ready" }, + error: { tone: "error", label: "Error" }, + removed: { tone: "neutral", label: "Removed" }, }; - const config = statusConfig[status]; - return ; + const config: { tone: StatusTone; label: string } = statusConfig[status] ?? { tone: "neutral", label: status }; + return ; }; const formatFileSize = (bytes?: number) => { diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx index ac790122507..54b05bc73b5 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx @@ -154,9 +154,8 @@ describe("VectorStoreTable", () => { it("should truncate long vector store IDs", () => { renderComponent(); - // Check that the truncated text is rendered (first 15 chars + ...) - const truncatedText = "very-long-vecto..."; - expect(screen.getByText(truncatedText)).toBeInTheDocument(); + const idButton = screen.getByText("very-long-vector-store-id-that-should-be-truncated"); + expect(idButton).toHaveClass("truncate", "max-w-[15ch]"); }); it("should make vector store ID clickable", async () => { @@ -245,13 +244,13 @@ describe("VectorStoreTable", () => { describe("Date Columns", () => { it("should render created at dates", () => { renderComponent(); - const dateElements = screen.getAllByText(/1\/\d+\/2024/); + const dateElements = screen.getAllByText(/Jan \d+, 2024/); expect(dateElements.length).toBe(6); // 3 created_at + 3 updated_at dates }); it("should render updated at dates", () => { renderComponent(); - const dateElements = screen.getAllByText(/1\/\d+\/2024/); + const dateElements = screen.getAllByText(/Jan \d+, 2024/); expect(dateElements.length).toBe(6); // 3 created_at + 3 updated_at dates }); }); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx index 180c2485ab5..c2e47cebe69 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx @@ -10,6 +10,7 @@ import { import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow } from "@tremor/react"; import { Tooltip } from "antd"; import React from "react"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import { getProviderLogoAndName } from "../provider_info_helpers"; import { VectorStore } from "./types"; @@ -28,19 +29,7 @@ const VectorStoreTable: React.FC = ({ data, onView, onEdi { header: "Vector Store ID", accessorKey: "vector_store_id", - cell: ({ row }) => { - const vectorStore = row.original; - return ( - - ); - }, + cell: ({ row }) => , }, { header: "Name", @@ -109,19 +98,13 @@ const VectorStoreTable: React.FC = ({ data, onView, onEdi header: "Created At", accessorKey: "created_at", sortingFn: "datetime", - cell: ({ row }) => { - const vectorStore = row.original; - return {new Date(vectorStore.created_at).toLocaleDateString()}; - }, + cell: ({ row }) => , }, { header: "Updated At", accessorKey: "updated_at", sortingFn: "datetime", - cell: ({ row }) => { - const vectorStore = row.original; - return {new Date(vectorStore.updated_at).toLocaleDateString()}; - }, + cell: ({ row }) => , }, { id: "actions", diff --git a/ui/litellm-dashboard/src/components/view_logs/audit_logs.tsx b/ui/litellm-dashboard/src/components/view_logs/audit_logs.tsx index 28b7afb6298..d811d3b9402 100644 --- a/ui/litellm-dashboard/src/components/view_logs/audit_logs.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/audit_logs.tsx @@ -4,7 +4,7 @@ import { Table, Tag, Input, Select, Button, Pagination, Spin } from "antd"; import { ReloadOutlined, LoadingOutlined } from "@ant-design/icons"; import type { ColumnsType } from "antd/es/table"; import { resolveLogoSrc } from "@/lib/assetPaths"; -import moment from "moment"; +import { DateCell, IdCell } from "@/components/shared/table_cells"; import { uiAuditLogsCall } from "../networking"; import { AuditLogEntry } from "./columns"; import { AuditLogDrawer } from "./AuditLogDrawer/AuditLogDrawer"; @@ -95,11 +95,7 @@ export default function AuditLogs({ userID, userRole, token, accessToken, isActi dataIndex: "updated_at", key: "updated_at", width: 200, - render: (val: string) => ( - - {moment.utc(val).local().format("MMM D, YYYY HH:mm:ss")} - - ), + render: (val: string) => , }, { title: "Action", @@ -123,7 +119,7 @@ export default function AuditLogs({ userID, userRole, token, accessToken, isActi title: "Object ID", dataIndex: "object_id", key: "object_id", - render: (val: string) => {val}, + render: (val: string) => , }, { title: "Changed By", @@ -137,7 +133,7 @@ export default function AuditLogs({ userID, userRole, token, accessToken, isActi dataIndex: "changed_by_api_key", key: "changed_by_api_key", width: 140, - render: (val: string) => (val ? {val.slice(0, 12)}… : "—"), + render: (val: string) => , }, ]; diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.test.tsx new file mode 100644 index 00000000000..afdd17813f3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/columns.test.tsx @@ -0,0 +1,56 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; + +import { createColumns, type LogEntry } from "./columns"; +import { DataTable } from "./table"; + +const logEntry = (overrides: Partial): LogEntry => ({ + request_id: "req-1", + api_key: "key-1", + team_id: "team-1", + model: "gpt-4o", + model_id: "model-1", + call_type: "acompletion", + spend: 0, + total_tokens: 10, + prompt_tokens: 5, + completion_tokens: 5, + startTime: "2026-07-07T09:50:13Z", + endTime: "2026-07-07T09:50:14Z", + cache_hit: "false", + messages: [], + response: {}, + ...overrides, +}); + +describe("Cost column", () => { + it("renders '-' for zero spend with no tooltip, so hovering never shows a contradictory $0", async () => { + const user = userEvent.setup(); + render( + r.request_id} + />, + ); + for (const dash of screen.getAllByText("-")) { + await user.hover(dash); + } + expect(screen.queryByText("$0")).not.toBeInTheDocument(); + }); + + it("shows the full-precision raw value in the tooltip for a real spend", async () => { + const user = userEvent.setup(); + render( + r.request_id} + />, + ); + const formatted = screen.getByText("$0.000123"); + await user.hover(formatted); + expect(await screen.findByText("$0.00012345678")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx index 7452992ed59..1d0f3f33d08 100644 --- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx @@ -1,11 +1,10 @@ +import { DateCell, IdCell, MoneyCell, StatusBadge } from "@/components/shared/table_cells"; import { getSpendString } from "@/utils/dataUtils"; import type { ColumnDef } from "@tanstack/react-table"; -import { Badge, Button } from "@tremor/react"; import { Tooltip } from "antd"; -import React, { useState } from "react"; +import React from "react"; import { getProviderLogoAndName } from "../provider_info_helpers"; import { TableHeaderSortDropdown } from "../common_components/TableHeaderSortDropdown/TableHeaderSortDropdown"; -import { TimeCell } from "./time_cell"; import { AGENT_CALL_TYPES, MCP_CALL_TYPES } from "./constants"; import { AgentBadge, AgentIcon, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges"; @@ -120,7 +119,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] : "Time", accessorKey: "startTime", size: 200, - cell: (info: any) => , + cell: (info: any) => , }, { header: "Type", @@ -174,48 +173,20 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] cell: (info: any) => { const status = info.getValue() || "Success"; const isSuccess = status.toLowerCase() !== "failure"; - - return ( - - {isSuccess ? "Success" : "Failure"} - - ); + return ; }, }, { header: "Session ID", accessorKey: "session_id", size: 120, - cell: (info: any) => { - const value = String(info.getValue() || ""); - const onSessionClick = info.row.original.onSessionClick; - return ( - - - - ); - }, + cell: (info: any) => , }, { header: "Request ID", accessorKey: "request_id", - cell: (info: any) => ( - - {String(info.getValue() || "")} - - ), + cell: (info: any) => , }, { header: sortProps @@ -236,11 +207,14 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] const row = info.row.original; const mcpCount = row.mcp_tool_call_count || 0; const mcpSpend = row.mcp_tool_call_spend || 0; + const spend = info.getValue(); return (
- - {getSpendString(info.getValue() || 0)} + + + + {mcpCount > 0 && mcpSpend > 0 && ( @@ -320,21 +294,7 @@ export const createColumns = (sortProps?: LogsSortProps): ColumnDef[] header: "Key Hash", accessorKey: "metadata.user_api_key", size: 110, - cell: (info: any) => { - const value = String(info.getValue() || "-"); - const onKeyHashClick = info.row.original.onKeyHashClick; - - return ( - - onKeyHashClick?.(value)} - > - {value} - - - ); - }, + cell: (info: any) => , }, { header: "Key Alias", @@ -589,141 +549,3 @@ export type AuditLogEntry = { before_value: Record; updated_values: Record; }; - -const getActionBadge = (action: string) => { - return ( - - {action} - - ); -}; - -export const auditLogColumns: ColumnDef[] = [ - { - id: "expander", - header: () => null, - cell: ({ row }) => { - const ExpanderCell = () => { - const [localExpanded, setLocalExpanded] = React.useState(row.getIsExpanded()); - - const toggleHandler = React.useCallback(() => { - setLocalExpanded((prev) => !prev); - row.getToggleExpandedHandler()(); - }, [row]); - - return row.getCanExpand() ? ( - - ) : ( - ● - ); - }; - return ; - }, - }, - { - header: "Timestamp", - accessorKey: "updated_at", - cell: (info: any) => , - }, - { - header: "Table Name", - accessorKey: "table_name", - cell: (info: any) => { - const tableName = info.getValue(); - let displayValue = tableName; - switch (tableName) { - case "LiteLLM_VerificationToken": - displayValue = "Keys"; - break; - case "LiteLLM_TeamTable": - displayValue = "Teams"; - break; - case "LiteLLM_OrganizationTable": - displayValue = "Organizations"; - break; - case "LiteLLM_UserTable": - displayValue = "Users"; - break; - case "LiteLLM_ProxyModelTable": - displayValue = "Models"; - break; - default: - displayValue = tableName; - } - return {displayValue}; - }, - }, - { - header: "Action", - accessorKey: "action", - cell: (info: any) => {getActionBadge(info.getValue())}, - }, - { - header: "Changed By", - accessorKey: "changed_by", - cell: (info: any) => { - const changedBy = info.row.original.changed_by; - const apiKey = info.row.original.changed_by_api_key; - return ( -
-
{changedBy}
- {apiKey && ( // Only show API key if it exists - -
- {" "} - {/* Apply max-width and truncate */} - {apiKey} -
- - )} -
- ); - }, - }, - { - header: "Affected Item ID", - accessorKey: "object_id", - cell: (props) => { - const ObjectIdDisplay = () => { - const objectId = props.getValue(); - const [copied, setCopied] = useState(false); - - if (!objectId) return <>-; - - const handleCopy = async () => { - try { - await navigator.clipboard.writeText(String(objectId)); - setCopied(true); - setTimeout(() => setCopied(false), 1500); - } catch (err) { - console.error("Failed to copy object ID: ", err); - } - }; - - return ( - - - {String(objectId)} - - - ); - }; - return ; - }, - }, -]; diff --git a/ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx b/ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx deleted file mode 100644 index 95a8b43b2c1..00000000000 --- a/ui/litellm-dashboard/src/components/view_logs/time_cell.test.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import { render, screen } from "@testing-library/react"; -import { describe, expect, it } from "vitest"; -import { TimeCell, getTimeZone } from "./time_cell"; - -describe("TimeCell", () => { - it("should render a formatted time string", () => { - render(); - // The global toLocaleString mock in setupTests returns "YYYY-MM-DD HH:MM:SS" - expect(screen.getByText(/2025/)).toBeInTheDocument(); - }); - - it("should render 'Error converting time' for invalid dates", () => { - // toLocaleString on an Invalid Date returns "Invalid Date", not throwing, - // but the component catches exceptions. Force an error by passing something - // that causes Date constructor to produce NaN. - render(); - // The mock returns "NaN-NaN-NaN NaN:NaN:NaN" for invalid dates - // The component has a try/catch that returns "Error converting time" on exception - const el = screen.getByText(/NaN|Error/); - expect(el).toBeInTheDocument(); - }); - - it("should render with monospace font", () => { - render(); - const span = screen.getByText(/2025/); - expect(span).toHaveStyle({ fontFamily: "monospace" }); - }); -}); - -describe("getTimeZone", () => { - it("should return a non-empty timezone string", () => { - const tz = getTimeZone(); - expect(typeof tz).toBe("string"); - expect(tz.length).toBeGreaterThan(0); - }); -}); diff --git a/ui/litellm-dashboard/src/components/view_logs/time_cell.tsx b/ui/litellm-dashboard/src/components/view_logs/time_cell.tsx deleted file mode 100644 index 8addc702dc6..00000000000 --- a/ui/litellm-dashboard/src/components/view_logs/time_cell.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import * as React from "react"; - -interface TimeCellProps { - utcTime: string; -} - -const getLocalTime = (utcTime: string): string => { - try { - const date = new Date(utcTime); - return date - .toLocaleString("en-US", { - year: "numeric", - month: "2-digit", - day: "2-digit", - hour: "2-digit", - minute: "2-digit", - second: "2-digit", - hour12: true, - }) - .replace(",", ""); - } catch (e) { - return "Error converting time"; - } -}; - -export const TimeCell: React.FC = ({ utcTime }) => { - return ( - - {getLocalTime(utcTime)} - - ); -}; - -export const getTimeZone = (): string => { - return Intl.DateTimeFormat().resolvedOptions().timeZone; -}; From 4a25cce114927f34287ec1c9444fc24199c1317c Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 14:57:21 -0700 Subject: [PATCH 028/101] fix(mcp): reject duplicate Authorization headers at MCP ingress For the client-forwarded token modes the gateway relays the caller's Authorization to the upstream, so a request carrying more than one Authorization header would make which token is forwarded ambiguous (the ASGI header list collapses to last-wins) and could diverge from what admission inspected. Multiple Authorization headers is malformed for bearer auth anyway (RFC 9110: not a comma-combinable field), so the ingress header converter now fails closed with a 400 instead of silently keeping one. Applies to every MCP request, not just passthrough. --- .../mcp_server/auth/user_api_key_auth_mcp.py | 31 +++++++++++++++- .../auth/test_user_api_key_auth_mcp.py | 35 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 9f28da19292..e7ffef0e4e3 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -591,10 +591,19 @@ class MCPRequestHandler: ASGI headers are in format: List[List[bytes, bytes]] We need to convert them to the format Headers expects. + + Collapsing the ASGI list into a dict keeps the last value for a duplicated + header name, so a request carrying more than one ``Authorization`` is + rejected first: for the client-forwarded token modes the gateway relays the + caller's ``Authorization`` upstream, so a duplicate would make which token is + forwarded ambiguous (and diverge from what admission inspected). Multiple + ``Authorization`` headers is malformed for bearer auth anyway (RFC 9110: not + a comma-combinable field), so fail closed with a 400. """ + raw_headers = scope.get("headers", []) + MCPRequestHandler._reject_duplicate_authorization(raw_headers) try: # ASGI headers are list of [name: bytes, value: bytes] pairs - raw_headers = scope.get("headers", []) # Convert bytes to strings and create dict for Headers constructor headers_dict = {name.decode("latin-1"): value.decode("latin-1") for name, value in raw_headers} return Headers(headers_dict) @@ -603,6 +612,26 @@ class MCPRequestHandler: # Return empty Headers object with empty dict return Headers({}) + @staticmethod + def _reject_duplicate_authorization(raw_headers: object) -> None: + """Raise 400 when the raw ASGI headers carry more than one ``Authorization`` header.""" + if not isinstance(raw_headers, (list, tuple)): + return + count = 0 + for entry in raw_headers: + if not isinstance(entry, (list, tuple)) or len(entry) < 1: + continue + name = entry[0] + if isinstance(name, (bytes, bytearray)) and bytes(name).lower() == b"authorization": + count += 1 + elif isinstance(name, str) and name.lower() == "authorization": + count += 1 + if count > 1: + raise HTTPException( + status_code=400, + detail="Multiple Authorization headers are not allowed", + ) + @staticmethod async def get_allowed_mcp_servers( user_api_key_auth: Optional[UserAPIKeyAuth] = None, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index c984ccb783e..ebaa6bc7cc0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -4,6 +4,7 @@ import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path @@ -750,6 +751,40 @@ class TestMCPRequestHandler: # For these tests, mcp_server_auth_headers should be empty assert mcp_server_auth_headers == {} + def test_duplicate_authorization_header_is_rejected(self): + """A request carrying more than one Authorization header is malformed for bearer auth and, + for the client-forwarded token modes, would make which upstream token is forwarded ambiguous. + The ingress header converter must reject it with a 400 rather than silently keeping one.""" + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/tp_server", + "headers": [ + (b"authorization", b"Bearer upstream-token-a"), + (b"authorization", b"Bearer upstream-token-b"), + (b"content-type", b"application/json"), + ], + } + with pytest.raises(HTTPException) as exc_info: + MCPRequestHandler._safe_get_headers_from_scope(scope) + assert exc_info.value.status_code == 400 + assert "Authorization" in str(exc_info.value.detail) + + def test_single_authorization_header_is_forwarded_verbatim(self): + """The rejection must not disturb the normal single-Authorization case: the value passes + through unchanged (guards against the duplicate check over-matching).""" + scope = { + "type": "http", + "method": "POST", + "path": "/mcp/tp_server", + "headers": [ + (b"authorization", b"Bearer upstream-token"), + (b"content-type", b"application/json"), + ], + } + headers = MCPRequestHandler._safe_get_headers_from_scope(scope) + assert headers.get("authorization") == "Bearer upstream-token" + @pytest.mark.asyncio class TestMCPOAuth2AuthFlow: From edf00bbe23c1c5be9363dc94f55ab837ca723005 Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 15:44:36 -0700 Subject: [PATCH 029/101] fix(mcp): recognize per-server auth header at the connect-time preemptive 401 The preemptive 401 for true_passthrough and oauth_delegate only inspected the request-wide Authorization, so a caller who bound the upstream token via the per-server x-mcp-{alias}-authorization header (the required shape in a multi-server aggregate, where the request-wide Authorization is withheld) was spuriously 401'd at initialize even though egress already honors that header. The gate now recognizes the per-server header for both modes via a shared helper, mode-correctly: true_passthrough treats any Authorization or the per-server header as the upstream token, oauth_delegate keeps requiring a distinct x-litellm-api-key so a lone Authorization consumed for admission is never mistaken for an upstream token. The preemptive raise is also gated to single-server scopes so a multi-server aggregate degrades gracefully instead of one missing token 401-ing the whole connect. --- .../proxy/_experimental/mcp_server/server.py | 64 +++++++++---- .../mcp_server/test_mcp_stale_session.py | 95 +++++++++++++++++++ 2 files changed, 139 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 4fda15f664e..d4597bf1adf 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1441,6 +1441,35 @@ if MCP_AVAILABLE: return allowed_mcp_servers + def _client_has_per_server_auth_header( + server: MCPServer, + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], + ) -> bool: + """True if the request carries a per-server ``x-mcp-{alias}-authorization`` + header for this server. This is the multi-server binding: it names one + upstream, so it is unambiguously the caller's upstream token regardless of + auth mode (never the LiteLLM admission credential). + """ + if not mcp_server_auth_headers: + return False + for key in (server.alias, server.server_name, server.name): + if not key: + continue + server_headers = None + for k, v in mcp_server_auth_headers.items(): + if k.lower() == key.lower(): + server_headers = v + break + if server_headers is None: + continue + if isinstance(server_headers, str) and server_headers.strip(): + return True + if isinstance(server_headers, dict): + for hk in server_headers.keys(): + if hk.lower() == "authorization": + return True + return False + def _client_has_passthrough_authorization( server: MCPServer, oauth2_headers: Optional[Dict[str, str]], @@ -1458,24 +1487,7 @@ if MCP_AVAILABLE: for k in oauth2_headers.keys(): if k.lower() == "authorization": return True - if mcp_server_auth_headers: - for key in (server.alias, server.server_name, server.name): - if not key: - continue - server_headers = None - for k, v in mcp_server_auth_headers.items(): - if k.lower() == key.lower(): - server_headers = v - break - if server_headers is None: - continue - if isinstance(server_headers, str) and server_headers.strip(): - return True - if isinstance(server_headers, dict): - for hk in server_headers.keys(): - if hk.lower() == "authorization": - return True - return False + return _client_has_per_server_auth_header(server, mcp_server_auth_headers) async def _get_user_oauth_extra_headers_from_db( server: MCPServer, @@ -3540,7 +3552,13 @@ if MCP_AVAILABLE: headers={"www-authenticate": www_authenticate}, ) - if server and server.is_oauth_delegate and _get_forwarded_auth_from_scope(scope) is None: + if ( + server + and server.is_oauth_delegate + and len(mcp_servers or []) == 1 + and _get_forwarded_auth_from_scope(scope) is None + and not _client_has_per_server_auth_header(server, mcp_server_auth_headers) + ): www_authenticate = _get_passthrough_www_authenticate( scope=scope, server_name=server_name, @@ -3551,7 +3569,13 @@ if MCP_AVAILABLE: headers={"www-authenticate": www_authenticate}, ) - if server and server.is_true_passthrough and not _scope_has_authorization_header(scope): + if ( + server + and server.is_true_passthrough + and len(mcp_servers or []) == 1 + and not _scope_has_authorization_header(scope) + and not _client_has_per_server_auth_header(server, mcp_server_auth_headers) + ): upstream_status, upstream_www_authenticate = await _probe_upstream_auth(server.url or "", "") if upstream_status == 401 and upstream_www_authenticate: raise HTTPException( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index 44ea5f43a70..04856b33f2a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -1216,6 +1216,101 @@ async def test_handle_streamable_http_mcp_oauth_delegate_with_forwarded_token_sk assert mock_handle_request.await_count == 1 +async def _run_passthrough_connect( + *, + auth_type, + server_names, + mcp_server_auth_headers, + scope_extra_headers=None, +): + """Drive handle_streamable_http_mcp through the preemptive-401 gate and report whether it + challenged (raised) or forwarded to the session manager. Returns (challenged, www_authenticate).""" + from litellm.proxy._experimental.mcp_server.server import ( + handle_streamable_http_mcp, + session_manager_stateless, + ) + + scope = _passthrough_mode_scope(server_names[0], extra_headers=scope_extra_headers) + receive = AsyncMock( + return_value={ + "type": "http.request", + "body": b'{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}', + "more_body": False, + } + ) + send = AsyncMock() + user_auth = MagicMock() + user_auth.user_id = "u1" + server = _build_passthrough_mode_server(server_names[0], auth_type) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", + new_callable=AsyncMock, + return_value=(user_auth, None, server_names, mcp_server_auth_headers, None, None), + ), + patch("litellm.proxy._experimental.mcp_server.server.set_auth_context"), + patch("litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True), + patch( + "litellm.proxy._experimental.mcp_server.server._handle_stale_mcp_session", + new_callable=AsyncMock, + return_value=False, + ), + patch( + "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager.get_mcp_server_by_name", + return_value=server, + ), + patch.object(session_manager_stateless, "handle_request", new_callable=AsyncMock) as mock_handle_request, + patch.object(session_manager_stateless, "_server_instances", {}), + ): + try: + await handle_streamable_http_mcp(scope, receive, send) + except HTTPException as exc: + return True, (exc.headers or {}).get("www-authenticate") + return mock_handle_request.await_count == 0, None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_type", [MCPAuth.oauth_delegate, MCPAuth.true_passthrough]) +async def test_handle_streamable_http_mcp_per_server_header_skips_preemptive_challenge(auth_type): + """A per-server x-mcp-{alias}-authorization header binds the upstream token to one server; the + connect gate must recognize it and forward instead of spuriously 401-ing, since egress already + honors it. Without this, the mandatory multi-server binding is unusable at connect.""" + try: + from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp # noqa: F401 + except ImportError: + pytest.skip("MCP server not available") + + challenged, _ = await _run_passthrough_connect( + auth_type=auth_type, + server_names=["pt_server"], + mcp_server_auth_headers={"pt_server": {"Authorization": "Bearer upstream-token"}}, + ) + assert challenged is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_type", [MCPAuth.oauth_delegate, MCPAuth.true_passthrough]) +async def test_handle_streamable_http_mcp_aggregate_does_not_preemptively_challenge(auth_type): + """A multi-server aggregate must degrade gracefully: the preemptive 401 is single-server only, so + one server missing a token cannot 401 the whole connect (the listing absorbs per-server failures).""" + try: + from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp # noqa: F401 + except ImportError: + pytest.skip("MCP server not available") + + challenged, _ = await _run_passthrough_connect( + auth_type=auth_type, + server_names=["pt_server", "pt_server_2"], + mcp_server_auth_headers=None, + ) + assert challenged is False + + @pytest.mark.asyncio async def test_handle_streamable_http_mcp_true_passthrough_without_token_surfaces_verbatim_upstream_challenge(): """true_passthrough is a transparent proxy: with no client Authorization the From ddec3b2b8b8846656611110c493307c3f5f70e53 Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 15:46:39 -0700 Subject: [PATCH 030/101] fix(mcp): plug fan-out Authorization bypass in the extra_headers loop The listing fan-out withholds the request-wide Authorization from a true_passthrough / oauth_delegate server when another server in scope also consumes it, so one bearer is not replayed across upstreams. The later server.extra_headers copy loop did not honor that decision: a server listing Authorization in extra_headers would re-copy the withheld bearer from raw_headers. The withhold decision is now computed once and applied to both the forwarding branch and the extra_headers loop. --- .../proxy/_experimental/mcp_server/server.py | 18 ++++++-- .../mcp_server/test_mcp_server.py | 43 +++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index d4597bf1adf..25890d29368 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1565,6 +1565,16 @@ if MCP_AVAILABLE: ) extra_headers: Optional[Dict[str, str]] = None + is_client_forwarded_mode = server.is_true_passthrough or server.is_oauth_delegate + # In a multi-server listing scope the request-wide Authorization can only carry one token, + # so it is withheld from a client-forwarded server when another server in scope also consumes + # it (RFC 9700 cross-resource replay); such scopes must bind per-server via + # x-mcp-{alias}-authorization. The decision is computed once so BOTH the forwarding branch and + # the extra_headers copy loop below honor it — otherwise a server that lists Authorization in + # extra_headers would re-copy the withheld bearer from raw_headers and replay it anyway. + withhold_forwarded_authorization = is_client_forwarded_mode and _caller_authorization_fans_out( + server, scope_servers + ) if server.auth_type == MCPAuth.oauth2: # For OAuth2 M2M servers, upstream Authorization must come from # client_credentials token fetch, never from caller headers. @@ -1583,8 +1593,8 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, ): extra_headers = _without_authorization(extra_headers) - elif server.is_true_passthrough or server.is_oauth_delegate: - if not _caller_authorization_fans_out(server, scope_servers): + elif is_client_forwarded_mode: + if not withhold_forwarded_authorization: extra_headers = _client_forwarded_authorization_headers( mcp_server=server, oauth2_headers=oauth2_headers, @@ -1611,7 +1621,9 @@ if MCP_AVAILABLE: for header in server.extra_headers: if not isinstance(header, str): continue - if header.lower() == "authorization" and strip_caller_authorization: + if header.lower() == "authorization" and ( + strip_caller_authorization or withhold_forwarded_authorization + ): continue header_value = normalized_raw_headers.get(header.lower()) if header_value is None: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 71ece75b2fb..12ebc31e33c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -426,6 +426,49 @@ def test_prepare_mcp_server_headers_scope_counts_legacy_delegate_as_consumer(): assert not extra_headers or "authorization" not in {k.lower() for k in extra_headers} +def test_prepare_mcp_server_headers_fanout_withhold_survives_extra_headers_loop(): + """Regression: when fan-out withholds the request-wide Authorization from a client-forwarded + server, the later server.extra_headers copy loop must not re-add it from raw_headers even if + the server lists Authorization in extra_headers. Otherwise one bearer is replayed across every + consuming upstream in the scope (the exact cross-resource replay the withholding prevents).""" + delegate = MCPServer( + server_id="od-extra-hdr", + name="od-extra-hdr", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + extra_headers=["Authorization"], + ) + second_consumer = _client_forwarded_mode_server("tp-peer", MCPAuth.true_passthrough) + + _, extra_headers = _prepare_headers_in_scope(delegate, [delegate, second_consumer]) + + assert not extra_headers or "authorization" not in {k.lower() for k in extra_headers} + + +def test_prepare_mcp_server_headers_sole_consumer_still_forwards_via_extra_headers(): + """Guard the fix does not over-withhold: with no second consumer in scope, a client-forwarded + server that lists Authorization in extra_headers still forwards the caller's bearer.""" + delegate = MCPServer( + server_id="od-extra-sole", + name="od-extra-sole", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth_delegate, + extra_headers=["Authorization"], + ) + static_server = MCPServer( + server_id="static-peer", + name="static-peer", + transport=MCPTransport.http, + auth_type=MCPAuth.api_key, + authentication_token="static-key", + ) + + _, extra_headers = _prepare_headers_in_scope(delegate, [delegate, static_server]) + + assert extra_headers is not None + assert extra_headers.get("Authorization") == "Bearer upstream-token" + + @pytest.mark.asyncio async def test_call_tool_m2m_skips_authorization_headers(): """M2M call_tool must not forward caller Authorization in oauth2/raw headers.""" From b2ea36f4f11ddc31122027ffb6eea225d5a485fd Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 16:22:05 -0700 Subject: [PATCH 031/101] fix(mcp): match sanitized per-server alias at the connect-time preemptive 401 The connect gate resolved x-mcp-{alias}-authorization by matching the raw lowercased alias/server_name/name only, but dashboard clients send x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization, and egress resolves those through lookup_mcp_server_auth_in_headers, which also tries the sanitized alias. So a per-server token bound with a sanitized alias (e.g. alias 'pt-server' arriving as header key 'pt_server') was forwarded at egress but still triggered a preemptive 401 at connect. _client_has_per_server_auth_header now resolves through the same lookup_mcp_server_auth_in_headers egress uses, so connect and egress agree on which header names match. --- .../proxy/_experimental/mcp_server/server.py | 32 +++++++++---------- .../mcp_server/test_mcp_stale_session.py | 19 +++++++++++ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 25890d29368..7e19c44052d 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -1449,25 +1449,25 @@ if MCP_AVAILABLE: header for this server. This is the multi-server binding: it names one upstream, so it is unambiguously the caller's upstream token regardless of auth mode (never the LiteLLM admission credential). + + Resolves through the same ``lookup_mcp_server_auth_in_headers`` egress uses, so + the connect gate and egress agree on which per-server header names match: a + dashboard client sends ``x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization``, + and matching only the raw alias here would 401 a token egress would forward. """ if not mcp_server_auth_headers: return False - for key in (server.alias, server.server_name, server.name): - if not key: - continue - server_headers = None - for k, v in mcp_server_auth_headers.items(): - if k.lower() == key.lower(): - server_headers = v - break - if server_headers is None: - continue - if isinstance(server_headers, str) and server_headers.strip(): - return True - if isinstance(server_headers, dict): - for hk in server_headers.keys(): - if hk.lower() == "authorization": - return True + from litellm.proxy._experimental.mcp_server.utils import ( + lookup_mcp_server_auth_in_headers, + ) + + server_headers = lookup_mcp_server_auth_in_headers( + mcp_server_auth_headers, alias=server.alias, server_name=server.server_name + ) + if isinstance(server_headers, str): + return bool(server_headers.strip()) + if isinstance(server_headers, dict): + return any(isinstance(hk, str) and hk.lower() == "authorization" for hk in server_headers) return False def _client_has_passthrough_authorization( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py index 04856b33f2a..da183e8d02a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_stale_session.py @@ -1293,6 +1293,25 @@ async def test_handle_streamable_http_mcp_per_server_header_skips_preemptive_cha assert challenged is False +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_type", [MCPAuth.oauth_delegate, MCPAuth.true_passthrough]) +async def test_handle_streamable_http_mcp_sanitized_per_server_header_skips_preemptive_challenge(auth_type): + """A dashboard client sends x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization, so the + alias 'pt-server' arrives as the header key 'pt_server'. Egress resolves that via the sanitized + alias, so the connect gate must too, or it 401s a token egress would forward.""" + try: + from litellm.proxy._experimental.mcp_server.server import handle_streamable_http_mcp # noqa: F401 + except ImportError: + pytest.skip("MCP server not available") + + challenged, _ = await _run_passthrough_connect( + auth_type=auth_type, + server_names=["pt-server"], + mcp_server_auth_headers={"pt_server": {"Authorization": "Bearer upstream-token"}}, + ) + assert challenged is False + + @pytest.mark.asyncio @pytest.mark.parametrize("auth_type", [MCPAuth.oauth_delegate, MCPAuth.true_passthrough]) async def test_handle_streamable_http_mcp_aggregate_does_not_preemptively_challenge(auth_type): From 5aa5b8ef32776770e8f51dbe7485a5ebba76ac91 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:30:46 -0700 Subject: [PATCH 032/101] fix(vertex): forward realtime health check params (#32550) * fix(vertex): forward realtime health check params * refactor(vertex): resolve realtime health check params via VertexBase helpers Address review feedback on the vertex param forwarding: pass model_params through to _realtime_health_check and resolve vertex credentials, project, and location inside the vertex_ai branch using the existing VertexBase.safe_get_vertex_ai_* helpers, so provider-specific key extraction no longer lives in litellm_core_utils and dict-typed vertex_credentials are supported * test(vertex): move realtime health check test to mapped unit test path codecov/patch reported the vertex branch of _realtime_health_check as uncovered because tests/litellm_utils_tests is not part of the unit test groups that upload coverage. Move the test into tests/test_litellm/litellm_core_utils/test_health_check_helpers.py, which the core-utils group runs, keeping the same end-to-end assertions through litellm.ahealth_check --------- Co-authored-by: Aleksandr Liadov <72351793+AleksandrLiadov@users.noreply.github.com> --- .../health_check_helpers.py | 1 + litellm/realtime_api/main.py | 12 ++-- .../test_health_check_helpers.py | 67 +++++++++++++++++++ 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 42ac82abf8b..9fc036e2a99 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -199,6 +199,7 @@ class HealthCheckHelpers: api_base=model_params.get("api_base", None), api_key=model_params.get("api_key", None), api_version=model_params.get("api_version", None), + model_params=model_params, ), "batch": lambda: HealthCheckHelpers._batch_health_check( custom_llm_provider=custom_llm_provider, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 0566ff73683..5ecf4d91ff6 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -515,6 +515,7 @@ async def _realtime_health_check( api_base: Optional[str] = None, api_version: Optional[str] = None, realtime_protocol: Optional[str] = None, + model_params: Optional[dict] = None, ): """ Health check for realtime API - tries connection to the realtime API websocket @@ -550,14 +551,17 @@ async def _realtime_health_check( elif custom_llm_provider == "xai": url = xai_realtime._construct_url(api_base=api_base or "https://api.x.ai/v1", query_params={"model": model}) elif custom_llm_provider == "vertex_ai": - vertex_location = litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION") - resolved_location = vertex_llm_base.get_vertex_region(vertex_region=vertex_location, model=model) + vertex_model_params = model_params or {} + resolved_location = vertex_llm_base.get_vertex_region( + vertex_region=VertexBase.safe_get_vertex_ai_location(vertex_model_params), + model=model, + ) ( access_token, resolved_project, ) = await vertex_llm_base._ensure_access_token_async( - credentials=None, - project_id=litellm.vertex_project or get_secret_str("VERTEXAI_PROJECT"), + credentials=VertexBase.safe_get_vertex_ai_credentials(vertex_model_params), + project_id=VertexBase.safe_get_vertex_ai_project(vertex_model_params), custom_llm_provider="vertex_ai", ) vertex_realtime_config = VertexAIRealtimeConfig( diff --git a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py index e8ef8f15142..f0d91224614 100644 --- a/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py +++ b/tests/test_litellm/litellm_core_utils/test_health_check_helpers.py @@ -277,3 +277,70 @@ async def test_batch_health_check_falls_back_to_acompletion_for_unsupported(): ) mock_alist.assert_not_called() mock_acompletion.assert_called_once_with(**model_params) + + +class _FakeWebsocketConnect: + def __init__(self, calls, url, **kwargs): + calls.append({"url": url, **kwargs}) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + +@pytest.mark.asyncio +async def test_realtime_health_check_uses_model_level_vertex_params(): + """Regression test: realtime health checks must resolve vertex_credentials, + vertex_project, and vertex_location from the model row's params instead of + falling back to process-global VERTEXAI_* settings.""" + import litellm + from litellm.realtime_api import main as realtime_main + + fake_vertex_base = MagicMock() + fake_vertex_base.get_vertex_region = MagicMock(return_value="us-central1") + fake_vertex_base._ensure_access_token_async = AsyncMock( + return_value=("model-level-token", "model-level-project") + ) + connect_calls = [] + + with ( + patch.object(realtime_main, "vertex_llm_base", fake_vertex_base), + patch( + "websockets.connect", + lambda url, **kwargs: _FakeWebsocketConnect(connect_calls, url, **kwargs), + ), + patch.object( + HealthCheckHelpers, + "_update_model_params_with_health_check_tracking_information", + staticmethod(lambda model_params: model_params), + ), + ): + result = await litellm.ahealth_check( + model_params={ + "model": "vertex_ai/gemini-live-2.5-flash-native-audio", + "vertex_credentials": '{"type":"service_account"}', + "vertex_project": "model-level-project", + "vertex_location": "us-central1", + }, + mode="realtime", + ) + + assert result == {} + fake_vertex_base.get_vertex_region.assert_called_once_with( + vertex_region="us-central1", model="gemini-live-2.5-flash-native-audio" + ) + fake_vertex_base._ensure_access_token_async.assert_called_once_with( + credentials='{"type":"service_account"}', + project_id="model-level-project", + custom_llm_provider="vertex_ai", + ) + assert connect_calls[0]["url"] == ( + "wss://us-central1-aiplatform.googleapis.com/ws/" + "google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" + ) + assert connect_calls[0]["additional_headers"] == { + "Authorization": "Bearer model-level-token", + "x-goog-user-project": "model-level-project", + } From e1b9ec1cd62ce436b863db1fa07179693592b364 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:37:38 -0700 Subject: [PATCH 033/101] feat(pricing): add xai/grok-4.5 model pricing and metadata (#32549) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 42 +++++++++++++++++++ model_prices_and_context_window.json | 42 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 70b6b05e6ec..db534b52df9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -38101,6 +38101,48 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-4.5": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4.5-latest": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-beta": { "input_cost_per_token": 5e-06, "litellm_provider": "xai", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b961c326625..363ba9842b0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -38303,6 +38303,48 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-4.5": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, + "xai/grok-4.5-latest": { + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "litellm_provider": "xai", + "max_input_tokens": 500000, + "max_output_tokens": 500000, + "max_tokens": 500000, + "mode": "chat", + "output_cost_per_token": 6e-06, + "output_cost_per_token_above_200k_tokens": 1.2e-05, + "source": "https://docs.x.ai/docs/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "xai/grok-beta": { "input_cost_per_token": 5e-06, "litellm_provider": "xai", From 9d745486d0b96d55e4a2809ea2cdea9465ddca2a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:48:03 -0700 Subject: [PATCH 034/101] fix(rerank): log optional_rerank_params at debug to stop leaking request content (#32533) * fix(rerank): log optional_rerank_params at debug not info to avoid leaking request content * test(rerank): exercise sync rerank path so coverage counts the log line --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/rerank_api/main.py | 2 +- tests/test_litellm/rerank_api/__init__.py | 0 tests/test_litellm/rerank_api/test_main.py | 67 ++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/rerank_api/__init__.py create mode 100644 tests/test_litellm/rerank_api/test_main.py diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 9320c7fae8a..b6ebf5589b7 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -158,7 +158,7 @@ def rerank( instruction=instruction, non_default_params=kwargs, ) - verbose_logger.info(f"optional_rerank_params: {optional_rerank_params}") + verbose_logger.debug(f"optional_rerank_params: {optional_rerank_params}") if isinstance(optional_params.timeout, str): optional_params.timeout = float(optional_params.timeout) diff --git a/tests/test_litellm/rerank_api/__init__.py b/tests/test_litellm/rerank_api/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/rerank_api/test_main.py b/tests/test_litellm/rerank_api/test_main.py new file mode 100644 index 00000000000..46d1461da50 --- /dev/null +++ b/tests/test_litellm/rerank_api/test_main.py @@ -0,0 +1,67 @@ +import logging +import os +import sys +from unittest.mock import MagicMock, patch + +sys.path.insert(0, os.path.abspath("../../..")) + +import litellm + +MARKER_QUERY = "MARKER_QUERY_do_not_log_at_info" +MARKER_DOC = "MARKER_DOC_sensitive_customer_text" + + +def _mock_cohere_response() -> MagicMock: + mock_response = MagicMock() + + def return_val(): + return { + "id": "cmpl-mockid", + "results": [{"index": 0, "relevance_score": 0.95}], + "meta": { + "api_version": {"version": "1.0"}, + "billed_units": {"search_units": 1}, + }, + } + + mock_response.json = return_val + mock_response.headers = {"key": "value"} + mock_response.status_code = 200 + return mock_response + + +def test_rerank_does_not_log_request_content_at_info(caplog): + """Regression for #32525: rerank must not emit query/documents to logs at INFO. + + The mapped ``optional_rerank_params`` (which always contains ``query`` and + ``documents``) bypasses ``turn_off_message_logging`` / ``redact_messages``, + so logging it at INFO leaks raw request content into stdout and any log sink. + """ + litellm.cohere_key = "test_api_key" + caplog.set_level(logging.DEBUG, logger="LiteLLM") + + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=_mock_cohere_response(), + ): + litellm.rerank( + model="cohere/rerank-english-v3.0", + query=MARKER_QUERY, + documents=[MARKER_DOC, "unrelated"], + top_n=2, + ) + + litellm_records = [r for r in caplog.records if r.name == "LiteLLM"] + + info_or_above = [ + r.getMessage() + for r in litellm_records + if r.levelno >= logging.INFO and (MARKER_QUERY in r.getMessage() or MARKER_DOC in r.getMessage()) + ] + assert not info_or_above, f"rerank leaked request content at INFO+: {info_or_above}" + + optional_params_logs = [r for r in litellm_records if "optional_rerank_params" in r.getMessage()] + assert optional_params_logs, "expected the optional_rerank_params line to be logged" + assert all( + r.levelno == logging.DEBUG for r in optional_params_logs + ), "optional_rerank_params must be logged at DEBUG, not INFO" From 637352735f2053e9326cf9c5c379548e2d00d7ce Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Jul 2026 18:35:05 -0700 Subject: [PATCH 035/101] fix(proxy): resolve team org from team_id so org admins can update team budgets An org admin updating a team budget from the Hub UI was rejected with 401, because the route gate only recognizes an org admin when the request body carries organization_id while the UI sends team_id. For /team/update, resolve the target team's organization_id from team_id before the gate runs, so an org admin of the team's own org clears the org-scoped branch without the client passing organization_id. Team admins and cross-org admins stay denied at the gate, and callers that already pass organization_id are unaffected, so the existing /team/update authorization matrix is unchanged --- litellm/proxy/auth/auth_checks.py | 25 +++- .../proxy/auth/auth_checks_organization.py | 32 ++++- .../management/test_team_update.py | 24 ++-- .../proxy/auth/test_route_checks.py | 129 ++++++++++++++++++ 4 files changed, 199 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index e7fee8d6eb2..cd8103abf5e 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -98,7 +98,10 @@ from litellm.repositories.user_repository import UserRepository from litellm.router import Router from litellm.utils import get_utc_datetime -from .auth_checks_organization import organization_role_based_access_check +from .auth_checks_organization import ( + add_team_org_context_to_request_body, + organization_role_based_access_check, +) from .auth_utils import get_model_from_request if TYPE_CHECKING: @@ -707,10 +710,28 @@ async def common_checks( # 10 [OPTIONAL] Organization RBAC checks organization_role_based_access_check(user_object=user_object, route=route, request_body=request_body) + async def _fetch_team_org_id(team_id: str) -> Optional[str]: + try: + team = await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + except HTTPException: + return None + return team.organization_id + + request_body_for_route_check = await add_team_org_context_to_request_body( + route=route, + request_body=request_body, + fetch_team_org_id=_fetch_team_org_id, + ) + _is_route_allowed = _is_api_route_allowed( route=route, request=request, - request_data=request_body, + request_data=request_body_for_route_check, valid_token=valid_token, user_obj=user_object, ) diff --git a/litellm/proxy/auth/auth_checks_organization.py b/litellm/proxy/auth/auth_checks_organization.py index 44c1d158cbe..b4caff9b8ee 100644 --- a/litellm/proxy/auth/auth_checks_organization.py +++ b/litellm/proxy/auth/auth_checks_organization.py @@ -2,7 +2,7 @@ Auth Checks for Organizations """ -from typing import Dict, List, Optional, Tuple +from typing import Awaitable, Callable, Dict, List, Optional, Tuple from fastapi import status @@ -170,3 +170,33 @@ def _user_is_org_admin( # User must be admin of ALL requested orgs, not just any one return all(org_id in admin_org_ids for org_id in candidate_org_ids) + + +TEAM_ORG_CONTEXT_ROUTES = frozenset({"/team/update"}) + + +async def add_team_org_context_to_request_body( + route: str, + request_body: dict, + fetch_team_org_id: Callable[[str], Awaitable[Optional[str]]], +) -> dict: + """ + Return a copy of request_body with organization_id resolved from the target + team when the route identifies the team by team_id and the caller did not + pass organization_id. This lets an org admin of the team's own org reach the + org-scoped branch of the route gate (which keys off organization_id) without + the client having to send it. Returns request_body unchanged when it does + not apply, so callers that already pass organization_id and non-team routes + are untouched. + """ + if route not in TEAM_ORG_CONTEXT_ROUTES: + return request_body + if request_body.get("organization_id"): + return request_body + team_id = request_body.get("team_id") + if not isinstance(team_id, str) or not team_id: + return request_body + org_id = await fetch_team_org_id(team_id) + if not org_id: + return request_body + return {**request_body, "organization_id": org_id} diff --git a/tests/proxy_behavior/management/test_team_update.py b/tests/proxy_behavior/management/test_team_update.py index 9cb2b0fecda..23ea89fa74d 100644 --- a/tests/proxy_behavior/management/test_team_update.py +++ b/tests/proxy_behavior/management/test_team_update.py @@ -105,27 +105,35 @@ async def test_team_update_authz_matrix( assert row.team_alias != MARKER_ALIAS, "denied but team mutated" -async def test_team_update_requires_proxy_admin_without_org_context( +async def test_team_update_org_admin_resolved_from_team_without_org_context( proxy_client, prisma, scratch, world ): - """With no organization_id in the body the route gate has no org context - and falls back to proxy-admin-only: an org admin of the team's own org - is 401, PROXY_ADMIN is 200.""" + """With no organization_id in the body the route gate resolves the target + team's org from team_id, so an org admin of the team's own org is allowed + (200), same as PROXY_ADMIN. A team admin of that same team stays denied + (401): the resolution grants org admins access, not team admins.""" await _seed_target(prisma, world, "alpha", scratch.prefix) - denied = await proxy_client.post( + allowed_org_admin = await proxy_client.post( "/team/update", headers={"Authorization": f"Bearer {world.keys[Actor.ORG_ADMIN].cleartext}"}, json={"team_id": scratch.prefix, "team_alias": MARKER_ALIAS}, ) - assert denied.status_code == 401, denied.text + assert allowed_org_admin.status_code == 200, allowed_org_admin.text - allowed = await proxy_client.post( + allowed_proxy_admin = await proxy_client.post( "/team/update", headers={"Authorization": f"Bearer {world.keys[Actor.PROXY_ADMIN].cleartext}"}, json={"team_id": scratch.prefix, "team_alias": MARKER_ALIAS}, ) - assert allowed.status_code == 200, allowed.text + assert allowed_proxy_admin.status_code == 200, allowed_proxy_admin.text + + denied_team_admin = await proxy_client.post( + "/team/update", + headers={"Authorization": f"Bearer {world.keys[Actor.TEAM_ADMIN].cleartext}"}, + json={"team_id": scratch.prefix, "team_alias": MARKER_ALIAS}, + ) + assert denied_team_admin.status_code == 401, denied_team_admin.text # Relocation gate — moving a team to a different org. The scratch team starts diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index d623149ff6a..204e6a671e3 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -2595,6 +2595,135 @@ def test_org_admin_of_multiple_orgs_can_operate_on_both(): assert _user_is_org_admin({"organizations": ["org-A", "org-B"]}, user_obj) is True +# ── LIT-4221: /team/update org-context resolution from team_id ──────────────── +from litellm.proxy.auth.auth_checks_organization import ( + add_team_org_context_to_request_body, +) + + +@pytest.mark.asyncio +async def test_add_team_org_context_resolves_org_from_team(): + """For /team/update with only team_id, the target team's org is resolved and + injected so the org-admin route gate can see it. This is what lets an org + admin update a team budget from the Hub UI, which sends team_id, not + organization_id (LIT-4221).""" + + async def fetch(team_id: str): + assert team_id == "team-1" + return "org-1" + + out = await add_team_org_context_to_request_body( + route="/team/update", + request_body={"team_id": "team-1", "max_budget": 42}, + fetch_team_org_id=fetch, + ) + assert out == {"team_id": "team-1", "max_budget": 42, "organization_id": "org-1"} + + +@pytest.mark.asyncio +async def test_add_team_org_context_noop_when_org_id_already_present(): + """If the caller already passed organization_id, no lookup happens and the + body is returned unchanged.""" + + async def fetch(team_id: str): + raise AssertionError("must not resolve when organization_id is present") + + body = {"team_id": "team-1", "organization_id": "org-explicit"} + out = await add_team_org_context_to_request_body( + route="/team/update", request_body=body, fetch_team_org_id=fetch + ) + assert out == body + + +@pytest.mark.asyncio +async def test_add_team_org_context_noop_for_other_routes(): + """Only /team/update opts into org resolution; other routes are untouched.""" + + async def fetch(team_id: str): + raise AssertionError("must not resolve for a non-opted-in route") + + body = {"team_id": "team-1"} + out = await add_team_org_context_to_request_body( + route="/team/delete", request_body=body, fetch_team_org_id=fetch + ) + assert out == body + + +@pytest.mark.asyncio +async def test_add_team_org_context_noop_when_team_has_no_org(): + """A standalone team (no org) resolves to None, so nothing is injected and + the org-admin branch stays unreachable (no blanket access).""" + + async def fetch(team_id: str): + return None + + body = {"team_id": "team-1"} + out = await add_team_org_context_to_request_body( + route="/team/update", request_body=body, fetch_team_org_id=fetch + ) + assert out == body + + +def test_team_update_gate_allows_org_admin_with_resolved_org(): + """Post-resolution (organization_id present), an org admin of that org clears + the gate for /team/update.""" + user_obj = _make_org_admin_user("org-1") + valid_token = UserAPIKeyAuth(user_id="org-admin-user", user_role=LitellmUserRoles.INTERNAL_USER.value) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/team/update", + request=request, + valid_token=valid_token, + request_data={"team_id": "team-1", "organization_id": "org-1"}, + ) + + +def test_team_update_gate_rejects_without_org_context(): + """Without organization_id (i.e. resolution found no org, or a non-org-admin), + the gate still rejects /team/update — the fix adds no blanket allow. Guards + against re-widening the route (e.g. dropping it into self_managed_routes).""" + user_obj = _make_org_admin_user("org-1") + valid_token = UserAPIKeyAuth(user_id="org-admin-user", user_role=LitellmUserRoles.INTERNAL_USER.value) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + + with pytest.raises(Exception): + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/team/update", + request=request, + valid_token=valid_token, + request_data={"team_id": "team-1", "max_budget": 42}, + ) + + +def test_team_update_gate_rejects_cross_org_admin_with_resolved_org(): + """Even after the target team's org is resolved, an org admin of a DIFFERENT + org is rejected at the gate (no cross-org escalation).""" + user_obj = _make_org_admin_user("org-1") + valid_token = UserAPIKeyAuth(user_id="org-admin-user", user_role=LitellmUserRoles.INTERNAL_USER.value) + request = MagicMock(spec=Request) + request.method = "POST" + request.query_params = {} + + with pytest.raises(Exception): + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/team/update", + request=request, + valid_token=valid_token, + request_data={"team_id": "team-1", "organization_id": "org-2"}, + ) + + @pytest.mark.asyncio async def test_initialize_pass_through_registers_wildcard_for_auth_subpath(): """ From febb27695b72e1aea5e9cfa4d3173291863361dc Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 8 Jul 2026 19:47:05 -0700 Subject: [PATCH 036/101] refactor(ui): point invitation links at the dedicated /onboarding route (#30857) * refactor(ui): point invitation links at the dedicated /onboarding route Invitation and reset-password links were built as /ui?invitation_id=..., which lands on the dashboard index and renders the onboarding form inline. They now point at the standalone /ui/onboarding route, so the index no longer has to special-case invitations. Old links keep working unchanged; the index still renders onboarding inline for ?invitation_id until the migration closeout removes that branch. Updates the three generators (the enterprise email builder, bulk user create, and the invitation/reset-password modal) and extracts the modal's URL building into a pure, unit-tested buildOnboardingUrl Refs LIT-3687 * refactor(ui): guard buildOnboardingUrl against a missing invitation id Return "" instead of emitting an invitation_id=undefined link when the id is not yet available, matching the existing empty-baseUrl guard. Placed after the SSO branch so the SSO link, which does not use the id, is unaffected Refs LIT-3687 --- .../send_emails/base_email.py | 4 +- .../send_emails/test_base_email.py | 27 +++++-- .../components/bulk_create_users_button.tsx | 2 +- .../src/components/onboarding_link.test.tsx | 70 +++++++++++++++++++ .../src/components/onboarding_link.tsx | 51 +++++++++----- 5 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/onboarding_link.test.tsx diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py index be80a12c80a..e7898cac565 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py @@ -919,9 +919,9 @@ class BaseEmailLogger(CustomLogger): """ Construct invitation link for the user - # http://localhost:4000/ui?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b + # http://localhost:4000/ui/onboarding?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b """ - return f"{base_url}/ui?invitation_id={invitation_id}" + return f"{base_url}/ui/onboarding?invitation_id={invitation_id}" async def send_email( self, diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py index c1ccc454305..61303340570 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py @@ -348,7 +348,9 @@ async def test_get_invitation_link(base_email_logger): result = await base_email_logger._get_invitation_link( user_id="test-user", base_url="http://test.com" ) - assert result == "http://test.com/ui?invitation_id=test-invitation-id" + assert ( + result == "http://test.com/ui/onboarding?invitation_id=test-invitation-id" + ) # Test with None user_id result = await base_email_logger._get_invitation_link( @@ -372,7 +374,7 @@ def test_construct_invitation_link(base_email_logger): result = base_email_logger._construct_invitation_link( invitation_id="test-id-123", base_url="http://test.com" ) - assert result == "http://test.com/ui?invitation_id=test-id-123" + assert result == "http://test.com/ui/onboarding?invitation_id=test-id-123" @pytest.mark.asyncio @@ -408,7 +410,10 @@ async def test_get_invitation_link_creates_new_when_none_exist(base_email_logger assert call_args["user_api_key_dict"].user_id == "test-user" # Verify the returned link uses the new invitation ID - assert result == "http://test.com/ui?invitation_id=new-invitation-id" + assert ( + result + == "http://test.com/ui/onboarding?invitation_id=new-invitation-id" + ) @pytest.mark.asyncio @@ -439,7 +444,10 @@ async def test_get_invitation_link_uses_existing_when_available(base_email_logge mock_create_invitation.assert_not_called() # Verify the returned link uses the existing invitation ID - assert result == "http://test.com/ui?invitation_id=existing-invitation-id" + assert ( + result + == "http://test.com/ui/onboarding?invitation_id=existing-invitation-id" + ) @pytest.mark.asyncio @@ -475,7 +483,10 @@ async def test_get_invitation_link_creates_new_when_list_is_none(base_email_logg assert call_args["user_api_key_dict"].user_id == "test-user" # Verify the returned link uses the new invitation ID - assert result == "http://test.com/ui?invitation_id=new-invitation-from-none" + assert ( + result + == "http://test.com/ui/onboarding?invitation_id=new-invitation-from-none" + ) @pytest.mark.asyncio @@ -495,7 +506,7 @@ async def test_get_email_params_user_invitation( with mock.patch.object( base_email_logger, "_get_invitation_link", - return_value="http://test.com/ui?invitation_id=test-id", + return_value="http://test.com/ui/onboarding?invitation_id=test-id", ): # Test with user invitation event result = await base_email_logger._get_email_params( @@ -509,7 +520,9 @@ async def test_get_email_params_user_invitation( == "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" ) assert result.support_contact == "support@berri.ai" - assert result.base_url == "http://test.com/ui?invitation_id=test-id" + assert ( + result.base_url == "http://test.com/ui/onboarding?invitation_id=test-id" + ) assert result.recipient_email == "test@example.com" diff --git a/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx b/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx index 17690c988dc..23b69722546 100644 --- a/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx +++ b/ui/litellm-dashboard/src/components/bulk_create_users_button.tsx @@ -361,7 +361,7 @@ const BulkCreateUsersButton: React.FC = ({ if (!uiSettings?.SSO_ENABLED) { // Regular invitation flow const invitationData = await invitationCreateCall(accessToken, user_id); - const invitationUrl = new URL(`/ui?invitation_id=${invitationData.id}`, baseUrl).toString(); + const invitationUrl = new URL(`/ui/onboarding?invitation_id=${invitationData.id}`, baseUrl).toString(); setParsedData((current) => current.map((u, i) => diff --git a/ui/litellm-dashboard/src/components/onboarding_link.test.tsx b/ui/litellm-dashboard/src/components/onboarding_link.test.tsx new file mode 100644 index 00000000000..039d5e250da --- /dev/null +++ b/ui/litellm-dashboard/src/components/onboarding_link.test.tsx @@ -0,0 +1,70 @@ +import { describe, it, expect } from "vitest"; +import { buildOnboardingUrl } from "./onboarding_link"; + +describe("buildOnboardingUrl", () => { + it("points the invitation link at the dedicated /ui/onboarding route", () => { + expect( + buildOnboardingUrl({ + baseUrl: "http://localhost:4000/", + invitationId: "inv-123", + hasUserSetupSso: false, + resetPassword: false, + }), + ).toBe("http://localhost:4000/ui/onboarding?invitation_id=inv-123"); + }); + + it("preserves a server_root_path prefix before /ui/onboarding", () => { + expect( + buildOnboardingUrl({ + baseUrl: "https://proxy.example.com/litellm", + invitationId: "inv-123", + hasUserSetupSso: false, + resetPassword: false, + }), + ).toBe("https://proxy.example.com/litellm/ui/onboarding?invitation_id=inv-123"); + }); + + it("appends action=reset_password for the reset-password flow", () => { + expect( + buildOnboardingUrl({ + baseUrl: "http://localhost:4000/", + invitationId: "inv-123", + hasUserSetupSso: false, + resetPassword: true, + }), + ).toBe("http://localhost:4000/ui/onboarding?invitation_id=inv-123&action=reset_password"); + }); + + it("sends SSO users to the dashboard root, not the onboarding form", () => { + expect( + buildOnboardingUrl({ + baseUrl: "http://localhost:4000/", + invitationId: "inv-123", + hasUserSetupSso: true, + resetPassword: false, + }), + ).toBe("http://localhost:4000/ui"); + }); + + it("returns an empty string when no base URL is known yet", () => { + expect( + buildOnboardingUrl({ + baseUrl: "", + invitationId: "inv-123", + hasUserSetupSso: false, + resetPassword: false, + }), + ).toBe(""); + }); + + it("returns an empty string rather than an invitation_id=undefined link when the id is not ready", () => { + expect( + buildOnboardingUrl({ + baseUrl: "http://localhost:4000/", + invitationId: undefined, + hasUserSetupSso: false, + resetPassword: false, + }), + ).toBe(""); + }); +}); diff --git a/ui/litellm-dashboard/src/components/onboarding_link.tsx b/ui/litellm-dashboard/src/components/onboarding_link.tsx index 7eb337a970a..0c27287a9e4 100644 --- a/ui/litellm-dashboard/src/components/onboarding_link.tsx +++ b/ui/litellm-dashboard/src/components/onboarding_link.tsx @@ -25,6 +25,32 @@ interface OnboardingProps { modalType?: "invitation" | "resetPassword"; } +export function buildOnboardingUrl({ + baseUrl, + invitationId, + hasUserSetupSso, + resetPassword, +}: { + baseUrl: string; + invitationId: string | undefined; + hasUserSetupSso: boolean; + resetPassword: boolean; +}): string { + if (!baseUrl) { + return ""; + } + const basePath = new URL(baseUrl).pathname; + const uiPath = basePath && basePath !== "/" ? `${basePath}/ui` : "ui"; + if (hasUserSetupSso) { + return new URL(uiPath, baseUrl).toString(); + } + if (!invitationId) { + return ""; + } + const action = resetPassword ? "&action=reset_password" : ""; + return new URL(`${uiPath}/onboarding?invitation_id=${invitationId}${action}`, baseUrl).toString(); +} + export default function OnboardingModal({ isInvitationLinkModalVisible, setIsInvitationLinkModalVisible, @@ -41,24 +67,13 @@ export default function OnboardingModal({ setIsInvitationLinkModalVisible(false); }; - const getInvitationUrl = () => { - if (!baseUrl) { - return ""; - } - const baseUrlObj = new URL(baseUrl); - const basePath = baseUrlObj.pathname; // This will be "/litellm" or "" - const path = basePath && basePath !== "/" ? `${basePath}/ui` : "ui"; - // Get the path from the base URL - if (invitationLinkData?.has_user_setup_sso) { - return new URL(path, baseUrl).toString(); - } - let urlPath = `${path}?invitation_id=${invitationLinkData?.id}`; - if (modalType === "resetPassword") { - urlPath += "&action=reset_password"; - } - const url = new URL(urlPath, baseUrl).toString(); - return url; - }; + const getInvitationUrl = () => + buildOnboardingUrl({ + baseUrl, + invitationId: invitationLinkData?.id, + hasUserSetupSso: invitationLinkData?.has_user_setup_sso ?? false, + resetPassword: modalType === "resetPassword", + }); return ( Date: Wed, 8 Jul 2026 20:54:40 -0700 Subject: [PATCH 037/101] fix(bedrock): honor cache_control ttl on message-level cachePoint blocks (#32551) Bedrock Converse supports cachePoint ttl (1h GA for Claude 4.5+), and _get_cache_point_block maps cache_control.ttl -> cachePoint.ttl, but the model parameter its allow-list gate requires was only threaded through the system-message path. Every message-level path either called _get_cache_point_block without model= (8 call sites in _bedrock_converse_messages_pt / _pt_async) or hardcoded CachePointBlock(type="default") (tool-result blocks and _convert_to_bedrock_tool_call_invoke), so a requested 1h ttl silently degraded to the 5-minute default - exactly on the conversation-tail breakpoint that long-running agents need to survive tool calls longer than 5 minutes. - pass model= at the 8 _get_cache_point_block call sites - tool-result blocks: capture the cache_control dict (was a boolean) and route through _get_cache_point_block so ttl survives - _convert_to_bedrock_tool_call_invoke: accept optional model and route per-tool-call cache_control through _get_cache_point_block Completes the ttl support added for system messages (#19848, #20326): message-level cache_control now behaves identically. Note: message-level cache_control on a content-less assistant message emits no cachePoint at all today; that pre-existing gap is orthogonal to ttl and left out of scope (per-tool-call placement covers it). Co-authored-by: Arash --- .../prompt_templates/factory.py | 56 +++++++---- .../chat/test_converse_transformation.py | 96 +++++++++++++++++++ 2 files changed, 135 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 06abb591717..8bb0e12905e 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3626,6 +3626,7 @@ class BedrockImageProcessor: def _convert_to_bedrock_tool_call_invoke( tool_calls: list, + model: Optional[str] = None, ) -> List[BedrockContentBlock]: """ OpenAI tool invokes: @@ -3701,7 +3702,13 @@ def _convert_to_bedrock_tool_call_invoke( # cache_control applies to the whole original # tool call; attach after the last split block. if tool.get("cache_control", None) is not None: - _parts_list.append(BedrockContentBlock(cachePoint=CachePointBlock(type="default"))) + _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( + {"cache_control": tool["cache_control"]}, + block_type="content_block", + model=model, + ) + if _cache_point_block is not None: + _parts_list.append(_cache_point_block) continue # Fallback: no objects extracted — use empty dict. arguments_dict = {} @@ -3712,8 +3719,13 @@ def _convert_to_bedrock_tool_call_invoke( # Check for cache_control and add a separate cachePoint block if tool.get("cache_control", None) is not None: - cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default")) - _parts_list.append(cache_point_block) + cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( + {"cache_control": tool["cache_control"]}, + block_type="content_block", + model=model, + ) + if cache_point_block is not None: + _parts_list.append(cache_point_block) return _parts_list except Exception as e: raise Exception( @@ -4417,22 +4429,27 @@ class BedrockConverseMessagesProcessor: tool_content.append(tool_call_result) # Check if we need to add a separate cachePoint block - has_cache_control = False + tool_msg_cache_control = None # Check for message-level cache_control if current_message.get("cache_control", None) is not None: - has_cache_control = True + tool_msg_cache_control = current_message["cache_control"] # Check for content-level cache_control in list content elif isinstance(current_message.get("content"), list): for content_element in current_message["content"]: if isinstance(content_element, dict) and content_element.get("cache_control", None) is not None: - has_cache_control = True + tool_msg_cache_control = content_element["cache_control"] break # Add a separate cachePoint block if cache_control is present - if has_cache_control: - cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default")) - tool_content.append(cache_point_block) + if tool_msg_cache_control is not None: + cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( + {"cache_control": tool_msg_cache_control}, + block_type="content_block", + model=model, + ) + if cache_point_block is not None: + tool_content.append(cache_point_block) msg_i += 1 # Deduplicate toolResult blocks with the same toolUseId @@ -4529,7 +4546,7 @@ class BedrockConverseMessagesProcessor: _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: - assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls)) + assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model)) msg_i += 1 @@ -4789,22 +4806,27 @@ def _bedrock_converse_messages_pt( tool_content.append(tool_call_result) # Check if we need to add a separate cachePoint block - has_cache_control = False + tool_msg_cache_control = None # Check for message-level cache_control if current_message.get("cache_control", None) is not None: - has_cache_control = True + tool_msg_cache_control = current_message["cache_control"] # Check for content-level cache_control in list content elif isinstance(current_message.get("content"), list): for content_element in current_message["content"]: if isinstance(content_element, dict) and content_element.get("cache_control", None) is not None: - has_cache_control = True + tool_msg_cache_control = content_element["cache_control"] break # Add a separate cachePoint block if cache_control is present - if has_cache_control: - cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default")) - tool_content.append(cache_point_block) + if tool_msg_cache_control is not None: + cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block( + {"cache_control": tool_msg_cache_control}, + block_type="content_block", + model=model, + ) + if cache_point_block is not None: + tool_content.append(cache_point_block) msg_i += 1 # Deduplicate toolResult blocks with the same toolUseId @@ -4902,7 +4924,7 @@ def _bedrock_converse_messages_pt( assistant_content.append(_cache_point_block) _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: - assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls)) + assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model)) msg_i += 1 diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index fe060e2e40d..9f2a4168dec 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -5671,3 +5671,99 @@ async def test_grounding_source_and_query_rendered_as_text(): user_content = result[0]["content"] assert {"text": "Tokyo is the capital of Japan."} in user_content assert {"text": "What is the capital of Japan?"} in user_content + + +def _agentic_messages_with_ttl(ttl_target: str): + """A tool-loop conversation with `ttl: 1h` cache_control at `ttl_target`: + 'user', 'tool_call' (per-tool-call, on the assistant's tool call), or + 'tool' (message-level, on the tool result - where + `cache_control_injection_points` with `index: -1` lands mid-loop). + + Message-level cache_control on a content-less assistant message emits no + cachePoint at all today (a separate gap, orthogonal to ttl); per-tool-call + placement covers that message, so it's excluded from the params below.""" + user: dict = {"role": "user", "content": "optimize this kernel " * 60} + assistant: dict = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "evaluate", "arguments": "{}"}, + } + ], + } + tool: dict = {"role": "tool", "tool_call_id": "call_1", "content": "score: 42"} + ttl_cc = {"type": "ephemeral", "ttl": "1h"} + if ttl_target == "user": + user["cache_control"] = ttl_cc + elif ttl_target == "tool_call": + assistant["tool_calls"][0]["cache_control"] = ttl_cc + elif ttl_target == "tool": + tool["cache_control"] = ttl_cc + return [user, assistant, tool] + + +def _collect_cache_points(result): + return [ + block["cachePoint"] + for message in result + for block in message.get("content") or [] + if "cachePoint" in block + ] + + +@pytest.mark.parametrize("ttl_target", ["user", "tool_call", "tool"]) +@pytest.mark.asyncio +async def test_message_level_cache_control_honors_ttl_for_supported_model( + ttl_target, +): + """Message- and tool-call-level cache_control must carry `ttl` onto the + emitted cachePoint for models that support extended caching, mirroring the + system-message path. Regression test for the gap left by the system-only + fix: the message paths called `_get_cache_point_block` without `model` (or + hardcoded `{"type": "default"}`), silently downgrading 1h to 5m.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + _bedrock_converse_messages_pt, + ) + + messages = _agentic_messages_with_ttl(ttl_target) + + result = _bedrock_converse_messages_pt( + messages=messages, + model="global.anthropic.claude-opus-4-7", + llm_provider="bedrock_converse", + ) + async_result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="global.anthropic.claude-opus-4-7", + llm_provider="bedrock_converse", + ) + ) + assert result == async_result + + cache_points = _collect_cache_points(result) + assert len(cache_points) == 1 + assert cache_points[0].get("ttl") == "1h" + + +@pytest.mark.parametrize("ttl_target", ["user", "tool_call", "tool"]) +def test_message_level_cache_control_drops_ttl_for_unsupported_model(ttl_target): + """Models outside the extended-caching allow-list must keep emitting the + plain `{"type": "default"}` cachePoint (Bedrock rejects `ttl` for them).""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + result = _bedrock_converse_messages_pt( + messages=_agentic_messages_with_ttl(ttl_target), + model="anthropic.claude-3-5-sonnet-20240620-v1:0", + llm_provider="bedrock_converse", + ) + + cache_points = _collect_cache_points(result) + assert len(cache_points) == 1 + assert "ttl" not in cache_points[0] From b4d63c1c9fd85eed13b1c7f47f311005c94f187b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 8 Jul 2026 22:36:13 -0700 Subject: [PATCH 038/101] ci: drop regex file guard from OSS daily guardrails The in-workflow regex list was hard to maintain and, because it runs on pull_request, could be modified by the same PR it inspects. Path gating for the OSS daily branches now lives in repository branch protection settings, so this workflow keeps only the OSS-safe checks: the hardcoded-secret test and ruff --- .github/workflows/oss_daily_guardrails.yml | 59 ---------------------- 1 file changed, 59 deletions(-) diff --git a/.github/workflows/oss_daily_guardrails.yml b/.github/workflows/oss_daily_guardrails.yml index 173b7cd4e41..950c51c9b60 100644 --- a/.github/workflows/oss_daily_guardrails.yml +++ b/.github/workflows/oss_daily_guardrails.yml @@ -17,65 +17,6 @@ concurrency: cancel-in-progress: true jobs: - sensitive-file-guard: - name: Block sensitive OSS daily changes - if: startsWith(github.ref_name, 'litellm_oss_daily_20') || startsWith(github.head_ref, 'litellm_oss_daily_20') || startsWith(github.base_ref, 'litellm_oss_daily_20') - runs-on: ubuntu-latest - timeout-minutes: 5 - - steps: - - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha || github.sha }} - persist-credentials: false - - - name: Check for sensitive file changes - env: - EVENT_NAME: ${{ github.event_name }} - BASE_REF_NAME: ${{ github.base_ref }} - HEAD_REF_NAME: ${{ github.head_ref }} - run: | - set -euo pipefail - - if [ "${EVENT_NAME}" = "pull_request" ] && echo "${HEAD_REF_NAME}" | grep -Eq '^litellm_oss_daily_20[0-9]{2}_[0-9]{2}_[0-9]{2}$'; then - # Final daily OSS branch PR into staging: review only the OSS delta - # accumulated on top of main, not unrelated main/staging drift. - BASE_REF="origin/main" - git fetch origin main - elif [ "${EVENT_NAME}" = "pull_request" ]; then - # PR targeting the daily OSS branch: review the incoming PR delta. - BASE_REF="origin/${BASE_REF_NAME}" - git fetch origin "${BASE_REF_NAME}" - else - # Push to the daily OSS branch: review the accumulated OSS delta. - BASE_REF="origin/main" - git fetch origin main - fi - - CHANGED_FILES="$(git diff --name-only "${BASE_REF}...HEAD")" - - if [ -z "${CHANGED_FILES}" ]; then - echo "No changed files detected." - exit 0 - fi - - echo "Changed files:" - echo "${CHANGED_FILES}" - - BLOCKED_FILES="$( - echo "${CHANGED_FILES}" | grep -E '(^\.github/workflows/|^\.github/actions/|^\.circleci/|^\.cursor/|^Dockerfile$|(^|/)Dockerfile$|^Makefile$|(^|/)Makefile$|(^|/)pyproject\.toml$|(^|/)uv\.lock$|(^|/)poetry\.lock$|(^|/)requirements[^/]*\.txt$|(^|/)package\.json$|(^|/)package-lock\.json$|(^|/)pnpm-lock\.yaml$|(^|/)yarn\.lock$|(^|/)tsconfig[^/]*\.json$|(^|/)(ruff|mypy|pytest|eslint|prettier|vitest|next|vite|jest)\.config\.)' || true - )" - - if [ -n "${BLOCKED_FILES}" ]; then - echo "::error::OSS daily branch contains sensitive workflow, config, dependency, or lockfile changes. Split these into a separate internal/security-reviewed PR." - echo "${BLOCKED_FILES}" - exit 1 - fi - - echo "No sensitive OSS daily file changes detected." - oss-safe-checks: name: Run OSS daily safe checks if: startsWith(github.ref_name, 'litellm_oss_daily_20') || startsWith(github.head_ref, 'litellm_oss_daily_20') || startsWith(github.base_ref, 'litellm_oss_daily_20') From 9813c4bf41b5b9b3af56c8f1dd4aa748bd98713b Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 11:58:29 +1000 Subject: [PATCH 039/101] feat(ui): add session id filter to request logs --- .../spend_management_endpoints.py | 8 ++ .../test_spend_management_endpoints.py | 84 +++++++++++++++++++ .../src/components/networking.tsx | 1 + .../components/view_logs/filter_options.ts | 5 ++ .../view_logs/log_filter_logic.test.tsx | 1 + .../components/view_logs/log_filter_logic.tsx | 4 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 7 files changed, 107 insertions(+) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 8f530e3b8ce..cf7bedfdc71 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1622,6 +1622,10 @@ async def ui_view_spend_logs( default=None, description="request_id to get spend logs for specific request_id", ), + session_id: str | None = fastapi.Query( + default=None, + description="Filter spend logs by session_id", + ), team_id: str | None = fastapi.Query( default=None, description="Filter spend logs by team_id", @@ -1772,6 +1776,9 @@ async def ui_view_spend_logs( if request_id is not None: where_conditions["request_id"] = request_id + if session_id is not None: + where_conditions["session_id"] = session_id + if model is not None: where_conditions["model"] = model @@ -1887,6 +1894,7 @@ async def ui_view_spend_logs( ('"user"', "user"), ("api_key", "api_key"), ("request_id", "request_id"), + ("session_id", "session_id"), ("model", "model"), ("model_id", "model_id"), ("model_group", "model_group"), diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 1e9818534c9..69ffc2275ff 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import collections import datetime import json import os @@ -85,6 +86,7 @@ def _reconstruct_ui_where_from_sql(sql_query, params): '"user"': "user", "api_key": "api_key", "request_id": "request_id", + "session_id": "session_id", "model": "model", "model_id": "model_id", "model_group": "model_group", @@ -162,7 +164,21 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No async def count(self, *args, **kwargs): return len(filter_fn(kwargs.get("where", {}))) + async def group_by(self, by, where, count): + allowed = set(where["session_id"]["in"]) + tallied = collections.Counter( + log["session_id"] + for log in mock_spend_logs + if log.get("session_id") in allowed + ) + return [ + {"session_id": sid, "_count": {"session_id": n}} + for sid, n in tallied.items() + ] + async def query_raw(self, sql_query, *params): + if "mcp_tool_call_count" in sql_query: + return [] filtered = filter_fn(_reconstruct_ui_where_from_sql(sql_query, params)) total = len(filtered) if "COUNT(*)" in sql_query: @@ -597,6 +613,74 @@ async def test_ui_view_spend_logs_with_user_id(client, monkeypatch): assert data["data"][0]["user"] == "test_user_1" +@pytest.mark.asyncio +async def test_ui_view_spend_logs_with_session_id(client, monkeypatch): + mock_spend_logs = [ + { + "id": "log1", + "request_id": "req1", + "api_key": "sk-test-key", + "user": "test_user_1", + "session_id": "session-abc", + "spend": 0.05, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + }, + { + "id": "log2", + "request_id": "req2", + "api_key": "sk-test-key", + "user": "test_user_1", + "session_id": "session-abc", + "spend": 0.10, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + }, + { + "id": "log3", + "request_id": "req3", + "api_key": "sk-test-key", + "user": "test_user_1", + "session_id": "session-other", + "spend": 0.02, + "startTime": datetime.datetime.now(timezone.utc).isoformat(), + "model": "gpt-4", + }, + ] + + def filter_by_session(where): + if "session_id" in where: + return [ + log + for log in mock_spend_logs + if log["session_id"] == where["session_id"] + ] + return mock_spend_logs + + monkeypatch.setattr( + "litellm.proxy.proxy_server.prisma_client", + make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_by_session), + ) + + start_date, end_date = _default_date_range() + + response = client.get( + "/spend/logs/ui", + params={ + "session_id": "session-abc", + "start_date": start_date, + "end_date": end_date, + }, + headers={"Authorization": "Bearer sk-test"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["total"] == 2 + assert {log["request_id"] for log in data["data"]} == {"req1", "req2"} + assert all(log["session_id"] == "session-abc" for log in data["data"]) + + # Mock spend logs with distinct values for sorting tests. # req_a: spend=0.10, tokens=500, start/end earliest # req_b: spend=0.05, tokens=200, start/end 2nd diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 403647106d4..5ec2765c621 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1929,6 +1929,7 @@ interface UiSpendLogsParams { api_key?: string; team_id?: string; request_id?: string; + session_id?: string; user_id?: string; end_user?: string; status_filter?: string; diff --git a/ui/litellm-dashboard/src/components/view_logs/filter_options.ts b/ui/litellm-dashboard/src/components/view_logs/filter_options.ts index 52632ea5861..90e27b6144d 100644 --- a/ui/litellm-dashboard/src/components/view_logs/filter_options.ts +++ b/ui/litellm-dashboard/src/components/view_logs/filter_options.ts @@ -63,6 +63,11 @@ export function getLogFilterOptions(accessToken: string): FilterOption[] { label: "Key Hash", isSearchable: false, }, + { + name: FILTER_KEYS.SESSION_ID, + label: "Session ID", + isSearchable: false, + }, { name: "Model", label: "Model", diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx index cbe37e0b70f..ef550baea91 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.test.tsx @@ -218,6 +218,7 @@ describe("useLogFilterLogic", () => { { filterKey: "Team ID", paramName: "team_id", value: "team-a" }, { filterKey: "Key Hash", paramName: "api_key", value: "key-x" }, { filterKey: "Request ID", paramName: "request_id", value: "req-xyz" }, + { filterKey: "Session ID", paramName: "session_id", value: "sess-42" }, { filterKey: "User ID", paramName: "user_id", value: "user-123" }, { filterKey: "End User", paramName: "end_user", value: "user-a" }, { filterKey: "Status", paramName: "status_filter", value: "error" }, diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index c45e03905d0..1d699042f05 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -30,6 +30,7 @@ export const FILTER_KEYS = { TEAM_ID: "Team ID", KEY_HASH: "Key Hash", REQUEST_ID: "Request ID", + SESSION_ID: "Session ID", MODEL: "Model", /** Exact match on LiteLLM_SpendLogs.model — use for search tools and public model names. */ PUBLIC_MODEL_OR_SEARCH_TOOL: "Public model / search tool", @@ -49,6 +50,7 @@ const TEXT_FILTER_KEYS: readonly (keyof LogFilterState)[] = [ FILTER_KEYS.KEY_HASH, FILTER_KEYS.ERROR_MESSAGE, FILTER_KEYS.REQUEST_ID, + FILTER_KEYS.SESSION_ID, FILTER_KEYS.USER_ID, FILTER_KEYS.PUBLIC_MODEL_OR_SEARCH_TOOL, ]; @@ -62,6 +64,7 @@ export const defaultFilters: LogFilterState = { [FILTER_KEYS.TEAM_ID]: "", [FILTER_KEYS.KEY_HASH]: "", [FILTER_KEYS.REQUEST_ID]: "", + [FILTER_KEYS.SESSION_ID]: "", [FILTER_KEYS.MODEL]: "", [FILTER_KEYS.PUBLIC_MODEL_OR_SEARCH_TOOL]: "", [FILTER_KEYS.USER_ID]: "", @@ -160,6 +163,7 @@ export function useLogFilterLogic({ api_key: effectiveFilters[FILTER_KEYS.KEY_HASH] || undefined, team_id: effectiveFilters[FILTER_KEYS.TEAM_ID] || undefined, request_id: effectiveFilters[FILTER_KEYS.REQUEST_ID] || undefined, + session_id: effectiveFilters[FILTER_KEYS.SESSION_ID] || undefined, user_id: effectiveFilters[FILTER_KEYS.USER_ID] || (filterByCurrentUser ? userID ?? undefined : undefined), end_user: effectiveFilters[FILTER_KEYS.END_USER] || undefined, status_filter: effectiveFilters[FILTER_KEYS.STATUS] || undefined, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 852f8388ec2..8e26729aa18 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -48633,6 +48633,8 @@ export interface operations { user_id?: string | null; /** @description request_id to get spend logs for specific request_id */ request_id?: string | null; + /** @description Filter spend logs by session_id */ + session_id?: string | null; /** @description Filter spend logs by team_id */ team_id?: string | null; /** @description Filter logs with spend greater than or equal to this value */ @@ -48739,6 +48741,8 @@ export interface operations { user_id?: string | null; /** @description request_id to get spend logs for specific request_id */ request_id?: string | null; + /** @description Filter spend logs by session_id */ + session_id?: string | null; /** @description Filter spend logs by team_id */ team_id?: string | null; /** @description Filter logs with spend greater than or equal to this value */ From f33403cb4b3b4122eadb2bb628ff7b99d8f5be02 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 12:39:13 +1000 Subject: [PATCH 040/101] feat(ui): support partial match on session id filter --- .../spend_management_endpoints.py | 12 ++- .../test_spend_management_endpoints.py | 91 +++++++++---------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index cf7bedfdc71..0bf35352d5f 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1624,7 +1624,7 @@ async def ui_view_spend_logs( ), session_id: str | None = fastapi.Query( default=None, - description="Filter spend logs by session_id", + description="Filter spend logs by session_id (partial string match)", ), team_id: str | None = fastapi.Query( default=None, @@ -1776,9 +1776,6 @@ async def ui_view_spend_logs( if request_id is not None: where_conditions["request_id"] = request_id - if session_id is not None: - where_conditions["session_id"] = session_id - if model is not None: where_conditions["model"] = model @@ -1894,7 +1891,6 @@ async def ui_view_spend_logs( ('"user"', "user"), ("api_key", "api_key"), ("request_id", "request_id"), - ("session_id", "session_id"), ("model", "model"), ("model_id", "model_id"), ("model_group", "model_group"), @@ -1914,6 +1910,12 @@ async def ui_view_spend_logs( p += 2 sql_conditions.append(or_clause) + if session_id is not None: + like_escaped_session_id = session_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + sql_conditions.append(f"session_id LIKE ${p}") + sql_params.append(f"%{like_escaped_session_id}%") + p += 1 + # Status filter if status_filter is not None: if status_filter == "success": diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 69ffc2275ff..c3899883d58 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -86,7 +86,6 @@ def _reconstruct_ui_where_from_sql(sql_query, params): '"user"': "user", "api_key": "api_key", "request_id": "request_id", - "session_id": "session_id", "model": "model", "model_id": "model_id", "model_group": "model_group", @@ -100,6 +99,7 @@ def _reconstruct_ui_where_from_sql(sql_query, params): alias = re.search(r"user_api_key_alias' LIKE \$(\d+)", cond) code = re.search(r"error_code' = \$(\d+)", cond) msg = re.search(r"error_message' LIKE \$(\d+)", cond) + sess = re.fullmatch(r"session_id LIKE \$(\d+)", cond) status = re.fullmatch(r"status = \$(\d+)", cond) if gte: date_bounds["gte"] = _iso(params[int(gte.group(1)) - 1]) @@ -109,6 +109,8 @@ def _reconstruct_ui_where_from_sql(sql_query, params): where["OR"] = where.get("OR", []) + [{"multi_team": True}] elif "status = 'success'" in cond: where["OR"] = where.get("OR", []) + [{"status": "success"}] + elif sess: + where["session_id"] = {"contains": str(params[int(sess.group(1)) - 1]).strip("%")} elif status: where["status"] = {"equals": params[int(status.group(1)) - 1]} elif alias: @@ -165,16 +167,14 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No return len(filter_fn(kwargs.get("where", {}))) async def group_by(self, by, where, count): - allowed = set(where["session_id"]["in"]) + col = by[0] + allowed = where.get(col, {}).get("in") tallied = collections.Counter( - log["session_id"] + log[col] for log in mock_spend_logs - if log.get("session_id") in allowed + if log.get(col) is not None and (allowed is None or log[col] in allowed) ) - return [ - {"session_id": sid, "_count": {"session_id": n}} - for sid, n in tallied.items() - ] + return [{col: value, "_count": {col: n}} for value, n in tallied.items()] async def query_raw(self, sql_query, *params): if "mcp_tool_call_count" in sql_query: @@ -614,48 +614,47 @@ async def test_ui_view_spend_logs_with_user_id(client, monkeypatch): @pytest.mark.asyncio -async def test_ui_view_spend_logs_with_session_id(client, monkeypatch): - mock_spend_logs = [ - { - "id": "log1", - "request_id": "req1", +@pytest.mark.parametrize( + "session_id_query,expected_request_ids", + [ + ("session-filter-demo-1", {"req1", "req2"}), + ("session-filter-demo-2", {"req3"}), + ("session-filter", {"req1", "req2", "req3"}), + ("demo", {"req1", "req2", "req3"}), + ("no-such-session", set()), + ], +) +async def test_ui_view_spend_logs_with_session_id( + client, monkeypatch, session_id_query, expected_request_ids +): + def make_log(request_id, session_id): + return { + "id": f"log-{request_id}", + "request_id": request_id, "api_key": "sk-test-key", "user": "test_user_1", - "session_id": "session-abc", + "session_id": session_id, "spend": 0.05, "startTime": datetime.datetime.now(timezone.utc).isoformat(), "model": "gpt-4", - }, - { - "id": "log2", - "request_id": "req2", - "api_key": "sk-test-key", - "user": "test_user_1", - "session_id": "session-abc", - "spend": 0.10, - "startTime": datetime.datetime.now(timezone.utc).isoformat(), - "model": "gpt-4", - }, - { - "id": "log3", - "request_id": "req3", - "api_key": "sk-test-key", - "user": "test_user_1", - "session_id": "session-other", - "spend": 0.02, - "startTime": datetime.datetime.now(timezone.utc).isoformat(), - "model": "gpt-4", - }, + } + + mock_spend_logs = [ + make_log("req1", "session-filter-demo-1"), + make_log("req2", "session-filter-demo-1"), + make_log("req3", "session-filter-demo-2"), + make_log("req4", "unrelated-abc"), ] def filter_by_session(where): - if "session_id" in where: - return [ - log - for log in mock_spend_logs - if log["session_id"] == where["session_id"] - ] - return mock_spend_logs + session_filter = where.get("session_id") + if session_filter is None: + return mock_spend_logs + return [ + log + for log in mock_spend_logs + if session_filter["contains"] in log["session_id"] + ] monkeypatch.setattr( "litellm.proxy.proxy_server.prisma_client", @@ -667,7 +666,7 @@ async def test_ui_view_spend_logs_with_session_id(client, monkeypatch): response = client.get( "/spend/logs/ui", params={ - "session_id": "session-abc", + "session_id": session_id_query, "start_date": start_date, "end_date": end_date, }, @@ -676,9 +675,9 @@ async def test_ui_view_spend_logs_with_session_id(client, monkeypatch): assert response.status_code == 200 data = response.json() - assert data["total"] == 2 - assert {log["request_id"] for log in data["data"]} == {"req1", "req2"} - assert all(log["session_id"] == "session-abc" for log in data["data"]) + assert data["total"] == len(expected_request_ids) + assert {log["request_id"] for log in data["data"]} == expected_request_ids + assert all(session_id_query in log["session_id"] for log in data["data"]) # Mock spend logs with distinct values for sorting tests. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8e26729aa18..4ca2f85be2b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -48633,7 +48633,7 @@ export interface operations { user_id?: string | null; /** @description request_id to get spend logs for specific request_id */ request_id?: string | null; - /** @description Filter spend logs by session_id */ + /** @description Filter spend logs by session_id (partial string match) */ session_id?: string | null; /** @description Filter spend logs by team_id */ team_id?: string | null; @@ -48741,7 +48741,7 @@ export interface operations { user_id?: string | null; /** @description request_id to get spend logs for specific request_id */ request_id?: string | null; - /** @description Filter spend logs by session_id */ + /** @description Filter spend logs by session_id (partial string match) */ session_id?: string | null; /** @description Filter spend logs by team_id */ team_id?: string | null; From 7801324ab0c06b1997010c14aed18b6a2d8fdce4 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 15:33:11 +1000 Subject: [PATCH 041/101] fix(spend): guard session_id filter against non-str query default --- litellm/proxy/spend_tracking/spend_management_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 0bf35352d5f..bce2e3581b5 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -1910,7 +1910,7 @@ async def ui_view_spend_logs( p += 2 sql_conditions.append(or_clause) - if session_id is not None: + if session_id is not None and isinstance(session_id, str): like_escaped_session_id = session_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") sql_conditions.append(f"session_id LIKE ${p}") sql_params.append(f"%{like_escaped_session_id}%") From 8a44fdd66321f26d87223112487e72e0a8c24e27 Mon Sep 17 00:00:00 2001 From: Thibault Serot Date: Thu, 9 Jul 2026 15:53:13 +1000 Subject: [PATCH 042/101] chore(ui): refresh eslint metrics for rebased base --- ui/litellm-dashboard/eslint-metrics.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index ded6ab97e1e..37bad071081 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,7 +1,7 @@ { "@typescript-eslint/no-explicit-any": 1982, "complexity": 128, - "local/no-large-inline-object-arg": 512, + "local/no-large-inline-object-arg": 519, "local/no-long-condition-chain": 233, "max-depth": 59, "no-console": 15 From e84a19acd566f6ac95ec6346ba603104adea728f Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 8 Jul 2026 23:24:11 -0700 Subject: [PATCH 043/101] fix(guardrails): walk Responses-API text taxonomy in shared content helpers (#32542) * fix(guardrails): walk Responses-API text taxonomy in shared content helpers Every guardrail sharing litellm/proxy/guardrails/_content_utils.py silently drops all text on the /v1/responses path. AIM turns it into a loud 422 ( {"error":"No messages in the request"}); every other guardrail (Lakera v2, Cato, Lasso, Repello, IBM, Azure Content Safety, enterprise secret detection) scans an empty payload and lets the request through unscanned. Three defects, all in _content_utils.py: 1. _iter_text_parts_in_content recognised only part.type == "text", but the Responses API uses input_text (request) and output_text (assistant). 2. _coerce_input_to_messages gated on "every item has a role key"; any Responses input list containing a function_call or function_call_output item failed the check and was wrapped as one opaque blob. 3. build_inspection_messages forwarded any role through, including a bare tool role missing tool_call_id, which validators like AIM's /fw/v1/analyze reject with a schema error. Fix walks the actual Responses item taxonomy (message, function_call, function_call_output, bare content parts and strings), recognises {text, input_text, output_text} everywhere, and coerces any role outside {system, user, assistant} to user in the outbound inspection payload. * style: ruff-format changed guardrail files * test(guardrails): cover function_call_output string form; drop em-dash in new docstring * fix(guardrails): map function_call_output straight to user role Avoids ever materialising a schema-invalid bare tool message. The downstream role-safety coercion in build_inspection_messages still guards genuinely caller-supplied non-standard roles (developer, function, custom values); add a regression test covering that path so the coercion has real coverage after this simplification. * test(guardrails): pin chat-completions tool-role coercion in build_inspection_messages * docs(test): soften AIM-specific claims in LIT-4294 test docstrings Ryan's review flagged that several test docstrings assert AIM's /fw/v1/analyze validates + rejects specific schema violations. That behavior is customer-reported in the LIT-4294 writeup, not directly verified by us. Rephrase to attribute the AIM 422 to the customer's writeup and describe the underlying constraint as the OpenAI chat schema; any downstream API that validates against that schema rejects the same shape. * refactor(guardrails): move unsupported-role coercion into AIM only The generic coercion in build_inspection_messages collapsed any role outside {system, user, assistant} to user for every caller of the helper. Combined with the pre-existing apply_redacted_messages_back write-back behavior in Lakera/AIM/Cato, that turned a loud OpenAI 400 on chat-completions tool-message masking into a silent semantic corruption of the outbound request (role tool with tool_call_id got rewritten to bare role user, dropping the assistant + tool_calls sibling). AIM specifically requires the coercion because its /fw/v1/analyze validates the payload against the OpenAI chat schema; other guardrails either do not validate roles or do their own reconstruction. Move the coercion to AimGuardrail._build_aim_inspection_messages so the shared helper keeps caller roles intact and no new cross-guardrail role corruption is introduced. The pre-existing apply_redacted_messages_back structural flatten remains as separate follow-up work. function_call_output items still synthesise role user in the shared helper because they have no natural role field, which is a different concern from coercing a caller-supplied role. * refactor(guardrails): preserve role fidelity in shared _content_utils Shared inspection helpers should extract text and preserve semantic role signals; role coercion for third-party schema safety stays inside the guardrail that needs it (AIM). Three shared-helper changes: - Bare content-part dicts (input_text/output_text) with an explicit role keep it; only role-less parts default to user. - Responses message items already had their role preserved; the behavior is now covered by an explicit test. - function_call_output items default to role tool (semantic equivalent of the chat-completions tool message shape) instead of role user, so Responses and chat completions produce symmetric inspection payloads. A caller-supplied role on the item is still preserved. AIM's schema-safe coercion in _build_aim_inspection_messages already handles the resulting role tool: it collapses to user before the POST to /fw/v1/analyze so AIM's OpenAI-schema validator does not reject the bare tool message (no tool_call_id can survive the flatten). Added a regression test in test_aim.py covering that path. --- litellm/proxy/guardrails/_content_utils.py | 60 ++--- .../guardrails/guardrail_hooks/aim/aim.py | 17 +- .../guardrails/guardrail_hooks/test_aim.py | 88 +++++++ .../proxy/guardrails/test_content_utils.py | 237 +++++++++++++++++- 4 files changed, 364 insertions(+), 38 deletions(-) create mode 100644 tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aim.py diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 1d31e33c77c..6cdf49818e6 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -32,6 +32,9 @@ def is_text_content_call_type(call_type: str) -> bool: return call_type in TEXT_CONTENT_CALL_TYPES +TEXT_PART_TYPES: FrozenSet[str] = frozenset({"text", "input_text", "output_text"}) + + def _iter_text_parts_in_content(content: Any) -> Iterator[str]: """Yield text fragments from a ``message.content`` value (string or multimodal list). Non-text parts (images, audio, …) are skipped.""" @@ -48,7 +51,7 @@ def _iter_text_parts_in_content(content: Any) -> Iterator[str]: continue if not isinstance(part, dict): continue - if part.get("type") == "text": + if part.get("type") in TEXT_PART_TYPES: text = part.get("text") if isinstance(text, str) and text: yield text @@ -58,14 +61,20 @@ def _coerce_input_to_messages(input_value: Any) -> List[Dict[str, Any]]: """Coerce a Responses-API ``data["input"]`` value into chat-style messages.""" if isinstance(input_value, str): return [{"role": "user", "content": input_value}] - if isinstance(input_value, list): - if input_value and all(isinstance(item, dict) and "role" in item for item in input_value): - return list(input_value) - # Mixed lists (content-part dicts + bare strings) and pure - # string/dict lists all become a single user message; the content - # iterator below handles each element type uniformly. - return [{"role": "user", "content": input_value}] - return [] + if not isinstance(input_value, list): + return [] + messages: List[Dict[str, Any]] = [] + for item in input_value: + if isinstance(item, str): + messages.append({"role": "user", "content": item}) + elif isinstance(item, dict): + if item.get("type") in TEXT_PART_TYPES: + messages.append({"role": item.get("role") or "user", "content": [item]}) + elif "content" in item: + messages.append({"role": item.get("role") or "user", "content": item["content"]}) + elif item.get("type") == "function_call_output" and "output" in item: + messages.append({"role": item.get("role") or "tool", "content": item["output"]}) + return messages def _iter_inspection_messages(data: Dict[str, Any]) -> Iterator[Dict[str, Any]]: @@ -112,7 +121,7 @@ def walk_user_text(data: Dict[str, Any], visit: Callable[[str], str]) -> int: new_parts.append(visit(part)) elif ( isinstance(part, dict) - and part.get("type") == "text" + and part.get("type") in TEXT_PART_TYPES and isinstance(part.get("text"), str) and part["text"] ): @@ -136,25 +145,20 @@ def walk_user_text(data: Dict[str, Any], visit: Callable[[str], str]) -> int: data["input"] = visit(input_value) return visited if isinstance(input_value, list): - # List of full messages: rewrite each message's content. - if input_value and all(isinstance(item, dict) and "role" in item for item in input_value): - for item in input_value: - if "content" in item: - item["content"] = _rewrite_content(item["content"]) - return visited - # List of content parts and/or bare strings: rewrite in place. for idx, item in enumerate(input_value): - if isinstance(item, str) and item: - visited += 1 - input_value[idx] = visit(item) - elif ( - isinstance(item, dict) - and item.get("type") == "text" - and isinstance(item.get("text"), str) - and item["text"] - ): - visited += 1 - input_value[idx] = {**item, "text": visit(item["text"])} + if isinstance(item, str): + if item: + visited += 1 + input_value[idx] = visit(item) + elif isinstance(item, dict): + if item.get("type") in TEXT_PART_TYPES: + if isinstance(item.get("text"), str) and item["text"]: + visited += 1 + input_value[idx] = {**item, "text": visit(item["text"])} + elif "content" in item: + item["content"] = _rewrite_content(item["content"]) + elif item.get("type") == "function_call_output" and "output" in item: + item["output"] = _rewrite_content(item["output"]) return visited return visited diff --git a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py index e7d9406ae3b..d22243cbe88 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py +++ b/litellm/proxy/guardrails/guardrail_hooks/aim/aim.py @@ -93,11 +93,10 @@ class AimGuardrail(CustomGuardrail): user_email=user_email, litellm_call_id=call_id, ) - # Covers multimodal list content + Responses-API input. response = await self.async_handler.post( f"{self.api_base}/fw/v1/analyze", headers=headers, - json={"messages": build_inspection_messages(data)}, + json={"messages": self._build_aim_inspection_messages(data)}, ) response.raise_for_status() res = response.json() @@ -116,6 +115,15 @@ class AimGuardrail(CustomGuardrail): verbose_proxy_logger.error(f"Aim: {action_type} action") return data + @staticmethod + def _build_aim_inspection_messages(data: dict) -> list[dict[str, str]]: + """AIM validates against the OpenAI chat schema. Bare ``role: "tool"`` + without ``tool_call_id`` and bare ``role: "function"`` without ``name`` + are rejected; the flatten drops those fields, so any role outside + ``{system, user, assistant}`` collapses to ``user`` for the AIM POST.""" + safe_roles = {"system", "user", "assistant"} + return [{**m, "role": "user"} if m["role"] not in safe_roles else m for m in build_inspection_messages(data)] + @staticmethod def _rejection(message: str, *, openai_code: str | None = None) -> ProxyException: return ProxyException( @@ -177,7 +185,10 @@ class AimGuardrail(CustomGuardrail): user_email=user_email, litellm_call_id=call_id, ), - json={"messages": build_inspection_messages(request_data) + [{"role": "assistant", "content": output}]}, + json={ + "messages": self._build_aim_inspection_messages(request_data) + + [{"role": "assistant", "content": output}] + }, ) response.raise_for_status() res = response.json() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aim.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aim.py new file mode 100644 index 00000000000..2e83422074e --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_aim.py @@ -0,0 +1,88 @@ +"""Tests for the AIM guardrail's inspection-payload construction.""" + +from litellm.proxy.guardrails.guardrail_hooks.aim.aim import AimGuardrail + + +def test_aim_inspection_messages_coerces_chat_completions_tool_role_to_user(): + """LIT-4294: A valid chat-completions ``role: "tool"`` message carries a + ``tool_call_id``, but the inspection flatten drops every field except + ``role`` and ``content``. A bare ``tool`` message without ``tool_call_id`` + is schema-invalid per the OpenAI chat schema, and the customer's writeup + reproduced AIM's ``/fw/v1/analyze`` returning 422 on exactly that shape. + The AIM POST collapses the role to ``user``; the outbound request to the + LLM is untouched.""" + data = { + "messages": [ + {"role": "user", "content": "weather in SF"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "user", "content": "weather in SF"}, + {"role": "user", "content": "sunny"}, + ] + + +def test_aim_inspection_messages_coerces_non_standard_caller_role_to_user(): + """LIT-4294: A caller-supplied role outside {system, user, assistant} + (e.g. ``developer``, ``function``) is coerced to ``user`` for the AIM + POST, since AIM validates the payload against the OpenAI chat schema + and rejects unknown roles the same way it rejects bare ``tool``.""" + data = { + "messages": [ + {"role": "developer", "content": "system-ish instruction"}, + {"role": "user", "content": "normal user text"}, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "user", "content": "system-ish instruction"}, + {"role": "user", "content": "normal user text"}, + ] + + +def test_aim_inspection_messages_coerces_responses_function_call_output_role(): + """LIT-4294: the shared helper synthesises ``role: "tool"`` for a + Responses ``function_call_output`` item (semantic equivalent of + chat-completions tool messages). AIM's schema-validating POST cannot + carry ``tool_call_id`` in the flat inspection payload, so AIM collapses + that ``tool`` role to ``user`` locally before POSTing.""" + data = { + "input": [ + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "sunny"}], + }, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "user", "content": "sunny"}, + ] + + +def test_aim_inspection_messages_preserves_safe_roles(): + """Safe roles pass through untouched — the coercion only fires for + roles the OpenAI chat schema flatten cannot represent standalone.""" + data = { + "messages": [ + {"role": "system", "content": "be helpful"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] + } + assert AimGuardrail._build_aim_inspection_messages(data) == [ + {"role": "system", "content": "be helpful"}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] diff --git a/tests/test_litellm/proxy/guardrails/test_content_utils.py b/tests/test_litellm/proxy/guardrails/test_content_utils.py index 099fca78a62..34d92505359 100644 --- a/tests/test_litellm/proxy/guardrails/test_content_utils.py +++ b/tests/test_litellm/proxy/guardrails/test_content_utils.py @@ -8,7 +8,6 @@ from litellm.proxy.guardrails._content_utils import ( walk_user_text, ) - # ── iter_message_text ──────────────────────────────────────────────────────────── @@ -101,6 +100,55 @@ def test_iter_message_text_empty_data(): assert list(iter_message_text({"input": ""})) == [] +def test_iter_message_text_responses_api_input_text_and_output_text_parts(): + """LIT-4294: Responses-API content parts use ``input_text`` (request) and + ``output_text`` (assistant); reading only ``type == "text"`` skipped every + ``/v1/responses`` body and every text guardrail was a no-op on that path.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "user text"}], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "assistant text"}], + }, + ] + } + assert list(iter_message_text(data)) == ["user text", "assistant text"] + + +def test_iter_message_text_responses_api_tool_call_taxonomy(): + """LIT-4294: a Responses ``input`` list freely mixes message items, + ``function_call`` (no ``role``), and ``function_call_output`` items. The + old ``all(item has 'role')`` gate wrapped the whole list as one blob and + yielded nothing; every text fragment must be visited independently.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "sunny"}], + }, + ] + } + assert list(iter_message_text(data)) == ["hello", "sunny"] + + # ── walk_user_text ──────────────────────────────────────────────────────────── @@ -160,6 +208,89 @@ def test_walk_user_text_redacts_responses_api_list_input(): assert data["input"][1] == {"type": "image_url", "image_url": {"url": "..."}} +def test_walk_user_text_redacts_responses_input_text_and_output_text_parts(): + """LIT-4294: ``walk_user_text`` must recognise the Responses text-part + variants so masking guardrails (secret detection, PII) actually redact + ``/v1/responses`` bodies instead of no-op'ing on them.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "AKIAEXAMPLE"}], + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "AKIAEXAMPLE too"}], + }, + ] + } + visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]")) + assert visited == 2 + assert data["input"][0]["content"][0] == { + "type": "input_text", + "text": "[REDACTED]", + } + assert data["input"][1]["content"][0] == { + "type": "output_text", + "text": "[REDACTED] too", + } + + +def test_walk_user_text_redacts_function_call_output_text(): + """LIT-4294: tool-call round-trips carry secrets in + ``function_call_output.output``; the redact walker must descend into it + while leaving ``function_call`` items (call_id, arguments) untouched.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "AKIAEXAMPLE user"}], + }, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": '{"AKIAEXAMPLE": 1}', + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "AKIAEXAMPLE tool"}], + }, + ] + } + visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]")) + assert visited == 2 + assert data["input"][0]["content"][0]["text"] == "[REDACTED] user" + assert data["input"][1] == { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": '{"AKIAEXAMPLE": 1}', + } + assert data["input"][2]["output"][0]["text"] == "[REDACTED] tool" + + +def test_walk_user_text_redacts_function_call_output_string_output(): + """LIT-4294: ``function_call_output.output`` is also a plain string in + OpenAI's Responses spec; the redact walker must handle both forms.""" + data = { + "input": [ + { + "type": "function_call_output", + "call_id": "c1", + "output": "AKIAEXAMPLE tool", + }, + ] + } + visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]")) + assert visited == 1 + assert data["input"][0]["output"] == "[REDACTED] tool" + + def test_walk_user_text_redacts_mixed_list_input(): """Read and write helpers must agree on coverage — bare strings inside a mixed ``input`` list are inspected by both.""" @@ -206,17 +337,13 @@ def test_build_inspection_messages_joins_multimodal_text_parts(): } ] } - assert build_inspection_messages(data) == [ - {"role": "user", "content": "first part\nsecond part"} - ] + assert build_inspection_messages(data) == [{"role": "user", "content": "first part\nsecond part"}] def test_build_inspection_messages_lifts_responses_api_input(): """fniVO9-F: ``input`` must be visible to hooks that POST messages to a remote API.""" data = {"input": "responses-api content"} - assert build_inspection_messages(data) == [ - {"role": "user", "content": "responses-api content"} - ] + assert build_inspection_messages(data) == [{"role": "user", "content": "responses-api content"}] def test_build_inspection_messages_drops_messages_with_no_text(): @@ -233,6 +360,102 @@ def test_build_inspection_messages_drops_messages_with_no_text(): assert build_inspection_messages(data) == [{"role": "user", "content": "kept"}] +def test_build_inspection_messages_responses_api_tool_call_taxonomy(): + """LIT-4294: mixed Responses ``input`` (message + function_call + + function_call_output) must produce a non-empty inspection list. The + customer's writeup reproduced a 422 from AIM's ``/fw/v1/analyze`` + (``No messages in the request``) when this synthesised list came back + empty; every other guardrail silently scanned nothing on the same + input.""" + data = { + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + }, + { + "type": "function_call", + "call_id": "c1", + "name": "get_weather", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "sunny"}], + }, + ] + } + assert build_inspection_messages(data) == [ + {"role": "user", "content": "hello"}, + {"role": "tool", "content": "sunny"}, + ] + + +def test_build_inspection_messages_function_call_output_defaults_to_tool(): + """LIT-4294: a Responses ``function_call_output`` item is the semantic + equivalent of a chat-completions ``role: "tool"`` message, so the shared + helper synthesises ``role: "tool"`` when the item has no explicit role. + AIM's schema-safe coercion happens at the AIM call site, not here.""" + data = { + "input": [ + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": "tool text"}], + }, + ] + } + assert build_inspection_messages(data) == [{"role": "tool", "content": "tool text"}] + + +def test_build_inspection_messages_function_call_output_preserves_explicit_role(): + """When ``function_call_output`` carries a caller-supplied ``role`` the + shared helper preserves it rather than synthesising ``tool``.""" + data = { + "input": [ + { + "type": "function_call_output", + "role": "assistant", + "call_id": "c1", + "output": [{"type": "input_text", "text": "tool text"}], + }, + ] + } + assert build_inspection_messages(data) == [{"role": "assistant", "content": "tool text"}] + + +def test_build_inspection_messages_bare_content_part_preserves_explicit_role(): + """A bare content-part dict with an explicit ``role`` keeps it. Only + absent roles get defaulted to ``user``.""" + data = { + "input": [ + {"type": "input_text", "text": "no role"}, + {"type": "output_text", "role": "assistant", "text": "with role"}, + ] + } + assert build_inspection_messages(data) == [ + {"role": "user", "content": "no role"}, + {"role": "assistant", "content": "with role"}, + ] + + +def test_build_inspection_messages_message_item_preserves_role(): + """Responses message items carry a role explicitly; the shared helper + passes it through untouched.""" + data = { + "input": [ + {"type": "message", "role": "system", "content": [{"type": "input_text", "text": "sys"}]}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "asst"}]}, + ] + } + assert build_inspection_messages(data) == [ + {"role": "system", "content": "sys"}, + {"role": "assistant", "content": "asst"}, + ] + + def test_build_inspection_messages_empty_data(): assert build_inspection_messages({}) == [] assert build_inspection_messages({"messages": []}) == [] From 1d870842125f52d901573cd0c2d6ba9c9399d39f Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Thu, 9 Jul 2026 10:51:37 +0300 Subject: [PATCH 044/101] refactor(otel): move litellm error detail keys under the litellm.* namespace (#32591) The v2 OTel integration stamped litellm-specific error details as error.code, error.stack_trace, and error.llm_provider, squatting on the semconv-owned error.* namespace. They now live at litellm.provider.error.code, litellm.provider.error.stack_trace, and litellm.provider.error.llm_provider alongside the other vendor-extension keys. error.type and error.message stay on the semconv keys. --- litellm/integrations/otel/model/semconv.py | 19 ++++++------- .../otel/test_otel_v2_components.py | 28 +++++++++---------- .../otel/test_otel_v2_sources_of_truth.py | 2 -- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 69d1e454655..aab80c7e5c4 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -147,24 +147,21 @@ class Error: """OTel-defined error attribute keys, from the semconv ``error.*`` registry. ``MESSAGE`` is marked *Deprecated* upstream in favor of domain-specific error message keys plus ``exception.message`` on the exception event, but - is still defined and stamped by litellm's v1 integration; keeping it here - for byte-for-byte parity.""" + litellm still stamps it.""" TYPE: Final = "error.type" MESSAGE: Final = "error.message" class LiteLLMError: - """LiteLLM-specific error attribute keys. Emitted under the ``error.*`` - namespace (not ``litellm.*``) for byte-for-byte compat with the v1 - integration in ``opentelemetry.py``; consumers reading these keys on v1 - spans read the same keys on v2 spans. OTel semconv does not define any of - these three, and per its extension rules a namespace may carry additional - vendor keys as long as they don't collide with defined names.""" + """Detail keys for the mapped provider exception of a failed LLM call. + OTel semconv does not define these, so they live under the ``litellm.*`` + vendor namespace rather than squatting on the semconv-owned ``error.*`` + namespace.""" - CODE: Final = "error.code" - STACK_TRACE: Final = "error.stack_trace" - LLM_PROVIDER: Final = "error.llm_provider" + CODE: Final = "litellm.provider.error.code" + STACK_TRACE: Final = "litellm.provider.error.stack_trace" + LLM_PROVIDER: Final = "litellm.provider.error.llm_provider" class ExceptionEvent: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 298047ec18b..eb795a64b79 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -604,8 +604,7 @@ def test_error_details_stamped_as_span_attributes_for_labels_ingest(): """OTel-defined keys and litellm-specific detail keys both ride span attributes so backends that flatten attrs into label indexes (Elastic APM ``labels.*``, Datadog span tags) render them. The exception event with the - full untruncated message stays alongside — both places, matching v1's - shape.""" + full untruncated message stays alongside.""" from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError from litellm.integrations.otel.emitter import SpanEmitter @@ -638,8 +637,8 @@ def test_error_details_stamped_as_span_attributes_for_labels_ingest(): # OTel-defined keys (from the ``error.*`` semconv registry). assert span.attributes[Error.TYPE] == "litellm.BadRequestError" assert span.attributes[Error.MESSAGE] == "400: violated moderation policy" - # LiteLLM-specific detail keys — vendor-namespaced under ``error.*`` - # for v1-parity, not defined by OTel semconv. + # LiteLLM-specific detail keys, under the ``litellm.provider.error.*`` + # vendor namespace, not defined by OTel semconv. assert span.attributes[LiteLLMError.CODE] == "400" assert span.attributes[LiteLLMError.STACK_TRACE] == "File proxy_server.py line 8570 ..." assert span.attributes[LiteLLMError.LLM_PROVIDER] == "openai" @@ -666,19 +665,18 @@ def test_error_details_omitted_when_span_error_carries_only_message(): assert LiteLLMError.LLM_PROVIDER not in span.attributes -def test_v2_error_attribute_keys_match_v1_error_attributes_byte_for_byte(): - """v1 (``opentelemetry.py``) and v2 (``otel/`` package) stamp identical - span-attribute keys so consumers reading ``labels.error_message`` don't - care which integration produced the span. Renaming either side is a - breaking change for downstream dashboards; this test locks the vocabulary.""" - from litellm.integrations._types.open_inference import ErrorAttributes +def test_error_attribute_keys_are_pinned(): + """``error.type`` and ``error.message`` come from the semconv ``error.*`` + registry; the litellm-specific detail keys are vendor keys under + ``litellm.provider.error.*``. Pins the exact strings so the emitted + vocabulary can't drift silently.""" from litellm.integrations.otel.model.semconv import Error, LiteLLMError - assert Error.TYPE == ErrorAttributes.ERROR_TYPE - assert Error.MESSAGE == ErrorAttributes.ERROR_MESSAGE - assert LiteLLMError.CODE == ErrorAttributes.ERROR_CODE - assert LiteLLMError.STACK_TRACE == ErrorAttributes.ERROR_STACK_TRACE - assert LiteLLMError.LLM_PROVIDER == ErrorAttributes.ERROR_LLM_PROVIDER + assert Error.TYPE == "error.type" + assert Error.MESSAGE == "error.message" + assert LiteLLMError.CODE == "litellm.provider.error.code" + assert LiteLLMError.STACK_TRACE == "litellm.provider.error.stack_trace" + assert LiteLLMError.LLM_PROVIDER == "litellm.provider.error.llm_provider" def test_error_message_falls_back_to_error_type_when_message_absent(): diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 89aa73a6066..71be28ea485 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -147,8 +147,6 @@ def test_attribute_keys_are_unique_across_namespaces(): from litellm.integrations.otel import MCP, Client, JsonRpc, LiteLLMError, Network # prefixes are allowed to be substrings; exact keys must not collide. - # ``LiteLLMError`` shares the ``error.*`` prefix with ``Error`` by design - # (v1-parity); the assert below is the guarantee they never overlap. exact = set() for cls in (GenAI, Error, LiteLLMError, Server, HTTP, DB, MCP, JsonRpc, Network, Client): for key in _all_constants(cls): From cda99a08c8eda814e573404d09e899a1f81b3646 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Thu, 9 Jul 2026 11:13:13 +0300 Subject: [PATCH 045/101] fix(proxy): surface OAuth error params in SSO callback (#32433) When an IdP denies SSO access it redirects back to /sso/callback with error and error_description query params and no code param. The callback previously fell through to the provider token exchange, which failed with a generic "'code' parameter was not found in callback request" 400 that hides the real denial reason. Raise a 401 that surfaces the IdP's error and description instead. Ported from #26640 with conflicts resolved against current staging --- litellm/proxy/management_endpoints/ui_sso.py | 12 ++++ .../proxy/management_endpoints/test_ui_sso.py | 60 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 43fdd3ed05a..065464aa565 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -1746,6 +1746,18 @@ async def auth_callback(request: Request, state: Optional[str] = None): """Verify login""" verbose_proxy_logger.info(f"Starting SSO callback with state: {state}") + oauth_error = request.query_params.get("error") + if oauth_error: + oauth_error_description = request.query_params.get("error_description") + verbose_proxy_logger.warning( + f"SSO callback received OAuth error: {oauth_error}, description: {oauth_error_description}" + ) + raise HTTPException( + status_code=401, + detail=f"OAuth error: {oauth_error}" + + (f", error_description: {oauth_error_description}" if oauth_error_description else ""), + ) + # Check if this is a CLI login (state starts with our CLI prefix) from litellm.constants import LITELLM_CLI_SESSION_TOKEN_PREFIX from litellm.proxy._types import LiteLLM_JWTAuth diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 642f20906a0..045e15f8b8b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -2783,6 +2783,7 @@ class TestCLIKeyRegenerationFlow: # Mock request mock_request = MagicMock(spec=Request) + mock_request.query_params = {"code": "some-auth-code"} cli_state = f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:cli-new-session-key-456" @@ -2824,6 +2825,7 @@ class TestCLIKeyRegenerationFlow: from litellm.proxy.management_endpoints.ui_sso import auth_callback mock_request = MagicMock(spec=Request) + mock_request.query_params = {"code": "some-auth-code"} cli_state = ( f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:cli-new-session-key-456:WXYZ-2345" ) @@ -7254,3 +7256,61 @@ async def test_cli_poll_key_tolerates_missing_user_row(): assert result["status"] == "ready" assert result["key"] == mock_jwt_token assert result["user_id"] == "just-created-user" + + +def _make_sso_callback_request(query_params: dict) -> MagicMock: + mock_request = MagicMock(spec=Request) + mock_request.query_params = query_params + return mock_request + + +@pytest.mark.asyncio +async def test_auth_callback_surfaces_oauth_error_with_description(): + """ + Regression: when the IdP denies access it redirects back with + ?error=...&error_description=... and no `code`. The callback must surface + that reason as a 401 instead of failing later on the missing `code` param. + """ + from litellm.proxy.management_endpoints.ui_sso import auth_callback + + mock_request = _make_sso_callback_request( + {"error": "access_denied", "error_description": "User is not assigned to the client application"} + ) + + with pytest.raises(HTTPException) as exc_info: + await auth_callback(request=mock_request, state=None) + + assert exc_info.value.status_code == 401 + assert "access_denied" in str(exc_info.value.detail) + assert "User is not assigned to the client application" in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_auth_callback_surfaces_oauth_error_without_description(): + """error_description is optional in the OAuth error response; the 401 detail must not render 'None'.""" + from litellm.proxy.management_endpoints.ui_sso import auth_callback + + mock_request = _make_sso_callback_request({"error": "access_denied"}) + + with pytest.raises(HTTPException) as exc_info: + await auth_callback(request=mock_request, state=None) + + assert exc_info.value.status_code == 401 + assert "access_denied" in str(exc_info.value.detail) + assert "None" not in str(exc_info.value.detail) + assert "error_description" not in str(exc_info.value.detail) + + +@pytest.mark.asyncio +async def test_auth_callback_without_oauth_error_proceeds_to_normal_flow(): + """Without an `error` query param the guard must not fire; the callback proceeds into the normal flow.""" + from litellm.proxy.management_endpoints.ui_sso import auth_callback + + mock_request = _make_sso_callback_request({"code": "some-auth-code"}) + + with patch("litellm.proxy.proxy_server.prisma_client", None): + with pytest.raises(HTTPException) as exc_info: + await auth_callback(request=mock_request, state=None) + + assert exc_info.value.status_code == 500 + assert "DB not connected" in str(exc_info.value.detail) From 60729f733ec7dd1d2a37826c3bb776e27daa6d11 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Thu, 9 Jul 2026 11:14:22 +0300 Subject: [PATCH 046/101] test(benchmarks): run shared logging executor inline to make CodSpeed measurements deterministic (#32435) --- tests/benchmarks/conftest.py | 36 +++++++++++++++++++++++++++++ tests/benchmarks/test_benchmarks.py | 21 +++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 tests/benchmarks/conftest.py diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 00000000000..c9b31cfb7d7 --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,36 @@ +"""Shared setup keeping CodSpeed measurements hermetic. + +CodSpeed's callgrind instrumentation counts instructions from every thread while +a measurement window is open, and valgrind serializes all threads onto one +virtual CPU. Work deferred to litellm's shared logging executor would therefore +be attributed to whichever benchmark the valgrind scheduler resumes it under, +flipping results between runs. Running the executor inline keeps each +benchmark's cost self-contained and deterministic. +""" + +from collections.abc import Callable, Iterator +from concurrent.futures import Future +from typing import ParamSpec, TypeVar + +import pytest + +from litellm.litellm_core_utils.thread_pool_executor import executor + +P = ParamSpec("P") +R = TypeVar("R") + + +def _submit_inline(fn: Callable[P, R], /, *args: P.args, **kwargs: P.kwargs) -> Future[R]: + future: Future[R] = Future() + try: + future.set_result(fn(*args, **kwargs)) + except BaseException as exc: + future.set_exception(exc) + return future + + +@pytest.fixture(autouse=True, scope="session") +def inline_logging_executor() -> Iterator[None]: + executor.submit = _submit_inline + yield + del executor.submit diff --git a/tests/benchmarks/test_benchmarks.py b/tests/benchmarks/test_benchmarks.py index 123dad93e11..59b3e0b6d5c 100644 --- a/tests/benchmarks/test_benchmarks.py +++ b/tests/benchmarks/test_benchmarks.py @@ -6,10 +6,13 @@ in the litellm hot path: token counting, model info lookup, provider resolution, and cost calculation. """ +import threading + import pytest import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.litellm_core_utils.token_counter import token_counter @@ -205,3 +208,21 @@ def test_get_model_cost_key_exact_match(): def test_get_model_cost_key_case_insensitive(): """Benchmark model cost key lookup with case-insensitive fallback.""" litellm.utils._get_model_cost_key("GPT-4o") + + +# --------------------------------------------------------------------------- +# Measurement hermeticity guard +# --------------------------------------------------------------------------- + + +@pytest.mark.benchmark +def test_logging_executor_runs_inline(): + """Guard that the shared logging executor runs submissions inline. + + Deferred submissions execute on worker threads, and callgrind attributes + their instructions to whichever benchmark's measurement window is open when + the valgrind scheduler resumes them, making results nondeterministic. + """ + future = executor.submit(threading.get_ident) + assert future.done() + assert future.result() == threading.get_ident() From c36525f9bef7549aa1810b50834ab131a7bcd531 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 9 Jul 2026 09:10:32 -0700 Subject: [PATCH 047/101] bump: litellm-enterprise 0.1.48 -> 0.1.49 --- enterprise/pyproject.toml | 4 ++-- pyproject.toml | 2 +- uv.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index b3864ce7878..85ccbef752f 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.48" +version = "0.1.49" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.48" +version = "0.1.49" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/pyproject.toml b/pyproject.toml index 3f5458c5494..99425ef3aef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ proxy = [ "azure-storage-blob>=12.28.0,<13.0", "mcp>=1.26.0,<2.0", "litellm-proxy-extras==0.4.75", - "litellm-enterprise==0.1.48", + "litellm-enterprise==0.1.49", "RestrictedPython>=8.1,<9.0", "rich>=13.9.4,<14.0", "polars>=1.38.1,<2.0", diff --git a/uv.lock b/uv.lock index e36da722261..a226ef172c7 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-04T17:13:14.93495Z" +exclude-newer = "2026-07-06T16:10:51.214184Z" exclude-newer-span = "P3D" [manifest] @@ -3639,7 +3639,7 @@ proxy-dev = [ [[package]] name = "litellm-enterprise" -version = "0.1.48" +version = "0.1.49" source = { editable = "enterprise" } [[package]] From b3a44bd1b2d46f2fd5f53a49107a6c1105901169 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:03:28 +0000 Subject: [PATCH 048/101] fix(deps): constrain soupsieve>=2.8.4 to patch two high-severity CVEs --- pyproject.toml | 1 + uv.lock | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3f5458c5494..d0ea2b9da4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -259,6 +259,7 @@ constraint-dependencies = [ "tornado>=6.5.6", "aiohttp>=3.14.1,<4.0", "packaging>=24.0", + "soupsieve>=2.8.4", ] override-dependencies = [ # a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0. diff --git a/uv.lock b/uv.lock index e36da722261..313fd682ccf 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-04T17:13:14.93495Z" +exclude-newer = "2026-07-06T17:03:18.138046444Z" exclude-newer-span = "P3D" [manifest] @@ -21,6 +21,7 @@ members = [ constraints = [ { name = "aiohttp", specifier = ">=3.14.1,<4.0" }, { name = "packaging", specifier = ">=24.0" }, + { name = "soupsieve", specifier = ">=2.8.4" }, { name = "tornado", specifier = ">=6.5.6" }, ] overrides = [{ name = "packaging", specifier = ">=24.0" }] @@ -7076,11 +7077,11 @@ wheels = [ [[package]] name = "soupsieve" -version = "2.8.3" +version = "2.8.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/ae/2d9c981590ed9999a0d91755b47fc74f74de286b0f5cee14c9269041e6c4/soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349", size = 118627, upload-time = "2026-01-20T04:27:02.457Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, + { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, ] [[package]] From ceeb90abdba02d502ee6391014a84954b406b852 Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 7 Jul 2026 20:17:35 -0700 Subject: [PATCH 049/101] feat(ui): expose true_passthrough and oauth_delegate auth types with a no-auth warning Adds the two client-forwarded token modes to the MCP server create and edit form auth dropdowns, and shows a warning when true_passthrough is selected: the gateway performs no admission auth for that server, so callers reach the upstream without a LiteLLM key and per-key/per-team rate limits and spend tracking do not apply. The warning is a shared component so the two forms cannot drift on the copy. --- .../mcp_tools/TruePassthroughWarning.tsx | 21 ++++++++++ .../mcp_tools/create_mcp_server.test.tsx | 25 ++++++++++++ .../mcp_tools/create_mcp_server.tsx | 7 ++++ .../mcp_tools/mcp_server_edit.test.tsx | 39 +++++++++++++++++++ .../components/mcp_tools/mcp_server_edit.tsx | 32 +++++++++------ .../src/components/mcp_tools/types.tsx | 2 + 6 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/TruePassthroughWarning.tsx diff --git a/ui/litellm-dashboard/src/components/mcp_tools/TruePassthroughWarning.tsx b/ui/litellm-dashboard/src/components/mcp_tools/TruePassthroughWarning.tsx new file mode 100644 index 00000000000..b52d3f4c672 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/TruePassthroughWarning.tsx @@ -0,0 +1,21 @@ +import React from "react"; +import { Alert } from "antd"; +import { AUTH_TYPE } from "./types"; + +/** + * Warning shown in the create/edit MCP server forms when auth_type + * true_passthrough is selected: the gateway performs no admission auth for + * that server, so callers reach the upstream without a LiteLLM identity. + */ +export default function TruePassthroughWarning({ authType }: { authType?: string | null }) { + if (authType !== AUTH_TYPE.TRUE_PASSTHROUGH) return null; + return ( + + ); +} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index 36af2f8d9fc..27e6d6a8e3d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -164,6 +164,31 @@ describe("CreateMCPServer", () => { }); }); + it("should warn that LiteLLM auth is disabled when True Passthrough is selected", async () => { + await selectHttpTransport(); + + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + await waitFor(() => { + expect( + screen.getByText("True Passthrough disables LiteLLM authentication for this server"), + ).toBeInTheDocument(); + }); + }); + + it("should not show the True Passthrough warning when OAuth Delegate is selected", async () => { + await selectHttpTransport(); + + await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)"); + + await waitFor(() => { + expect(screen.getAllByText("OAuth Delegate (client-supplied upstream token)").length).toBeGreaterThan(0); + }); + expect( + screen.queryByText("True Passthrough disables LiteLLM authentication for this server"), + ).not.toBeInTheDocument(); + }); + it("should not require auth value when creating a server with API Key auth type", async () => { await selectHttpTransport(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 10668468c15..1362835f475 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -16,6 +16,7 @@ import { MCP_OAUTH2_FLOW_INTERACTIVE, } from "./types"; import OAuthFormFields from "./OAuthFormFields"; +import TruePassthroughWarning from "./TruePassthroughWarning"; import TokenExchangeFormFields from "./TokenExchangeFormFields"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPConnectionStatus from "./mcp_connection_status"; @@ -970,9 +971,15 @@ const CreateMCPServer: React.FC = ({ OAuth OAuth Token Exchange (OBO) AWS SigV4 (Bedrock AgentCore MCPs) + True Passthrough (no LiteLLM auth) + + OAuth Delegate (client-supplied upstream token) + + + {shouldShowAuthValueField && ( { }); }); +describe("MCPServerEdit (true passthrough warning)", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + const renderWithAuthType = (authType: string) => + render( + , + ); + + it("warns that LiteLLM auth is disabled for a true_passthrough server", async () => { + renderWithAuthType("true_passthrough"); + + await waitFor(() => { + expect(screen.getByText("True Passthrough disables LiteLLM authentication for this server")).toBeInTheDocument(); + }); + }); + + it("does not warn for an oauth_delegate server", async () => { + renderWithAuthType("oauth_delegate"); + + await waitFor(() => { + expect(screen.getAllByRole("button", { name: "Save Changes" }).length).toBeGreaterThan(0); + }); + expect( + screen.queryByText("True Passthrough disables LiteLLM authentication for this server"), + ).not.toBeInTheDocument(); + }); +}); + describe("MCPServerEdit (auth type switch)", () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 70632b459fc..2c4f674c14b 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -18,6 +18,7 @@ import { getToken, isTokenValid, setToken } from "@/utils/mcpTokenStore"; import { buildMcpPassthroughAuthHeader } from "@/utils/mcpHeaderUtils"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPPermissionManagement from "./MCPPermissionManagement"; +import TruePassthroughWarning from "./TruePassthroughWarning"; import MCPToolConfiguration from "./mcp_tool_configuration"; import StdioConfiguration from "./StdioConfiguration"; import TokenExchangeFormFields from "./TokenExchangeFormFields"; @@ -874,18 +875,25 @@ const MCPServerEdit: React.FC = ({ {/* Authentication - for HTTP, SSE, and OpenAPI */} {!isStdioTransport && ( - - - + <> + + + + + )} {isStdioTransport && ( diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 9469d7bd89e..70cc8129bf1 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -41,6 +41,8 @@ export const AUTH_TYPE = { OAUTH2: "oauth2", OAUTH2_TOKEN_EXCHANGE: "oauth2_token_exchange", AWS_SIGV4: "aws_sigv4", + TRUE_PASSTHROUGH: "true_passthrough", + OAUTH_DELEGATE: "oauth_delegate", }; export const OAUTH_FLOW = { From 22ab518071716688f9b0f70003fb30780ca6ff42 Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 7 Jul 2026 21:29:43 -0700 Subject: [PATCH 050/101] feat(ui): browser-only Authorize & Fetch for the client-forwarded token modes true_passthrough and oauth_delegate persist no upstream credentials, so the create/edit forms had no way to preview tools or configure the tool allowlist: tools/list went upstream unauthenticated and came back 401. This reuses the existing OAuth authorize machinery in browser-only mode for those two auth types: the admin authorizes against the upstream (DCR/PKCE, with optional client credentials for IdPs without dynamic registration), the token lands in sessionStorage exactly like the legacy PKCE-passthrough path, and the tools preview forwards it via the per-server x-mcp-{alias}-authorization header, which the passthrough resolver arm already accepts. Nothing is written to the server row or the per-user credential store; the create payload keeps excluding credentials for these auth types via AUTH_TYPES_REQUIRING_CREDENTIALS. The tools preview endpoint now also extracts the Authorization header for the two new auth types so the browser-held token reaches the passthrough arm during create-time previews. --- .../mcp_server/rest_endpoints.py | 6 +- .../mcp_server/test_rest_endpoints.py | 53 +++++++++++++ .../mcp_tools/PassthroughAuthorizeSection.tsx | 74 +++++++++++++++++++ .../mcp_tools/create_mcp_server.test.tsx | 26 +++++++ .../mcp_tools/create_mcp_server.tsx | 11 +++ .../mcp_tools/mcp_server_edit.test.tsx | 49 ++++++++++++ .../components/mcp_tools/mcp_server_edit.tsx | 66 +++++++++++++---- .../src/hooks/useTestMCPConnection.tsx | 4 +- 8 files changed, 272 insertions(+), 17 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index b917530dd52..21682b4dd3e 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1322,7 +1322,11 @@ if MCP_AVAILABLE: mcp_auth_header = credentials.get("auth_value") oauth2_headers: Optional[Dict[str, str]] = None - if new_mcp_server_request.auth_type == MCPAuth.oauth2: + if new_mcp_server_request.auth_type in { + MCPAuth.oauth2, + MCPAuth.true_passthrough, + MCPAuth.oauth_delegate, + }: oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers) async def _list_tools_operation(client): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 3d9afd8f250..090b4711dc9 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -593,6 +593,59 @@ class TestTestToolsList: assert captured["oauth2_headers"] == oauth_headers assert oauth_call_counter["count"] == 1 + @pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) + async def test_extracts_oauth2_headers_for_client_forwarded_modes(self, monkeypatch, auth_type): + """The browser-only authorize flow sends the upstream token as Authorization; the preview + must thread it through for the client-forwarded token modes so the passthrough arm can + forward it, instead of probing the upstream unauthenticated.""" + + captured: dict = {} + + async def fake_execute( + request, + operation, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers=None, + ): + captured["mcp_auth_header"] = mcp_auth_header + captured["oauth2_headers"] = oauth2_headers + return { + "tools": [], + "error": None, + "message": "Successfully retrieved tools", + } + + monkeypatch.setattr(rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False) + + oauth_headers = {"Authorization": "Bearer upstream-token"} + + monkeypatch.setattr( + auth_mcp.MCPRequestHandler, + "_get_oauth2_headers_from_headers", + staticmethod(lambda headers: oauth_headers), + raising=False, + ) + + request = _build_request({"authorization": "Bearer upstream-token"}) + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=auth_type, + ) + + from litellm.proxy._types import LitellmUserRoles + + result = await rest_endpoints.test_tools_list( + request, + payload, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert result["message"] == "Successfully retrieved tools" + assert captured["mcp_auth_header"] is None + assert captured["oauth2_headers"] == oauth_headers + class TestListToolsRestAPI: pytestmark = pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx new file mode 100644 index 00000000000..453e3024000 --- /dev/null +++ b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx @@ -0,0 +1,74 @@ +import React from "react"; +import { Button, Form, Input } from "antd"; +import { AUTH_TYPE } from "./types"; + +interface PassthroughOAuthFlow { + startOAuthFlow: () => void | Promise; + status: string; + error: string | null; + tokenResponse: { access_token?: string; expires_in?: number } | null; +} + +/** + * Browser-only Authorize & Fetch for the client-forwarded token modes + * (true_passthrough / oauth_delegate). LiteLLM never stores upstream + * credentials for these modes, so the token obtained here lives in this + * browser session only: it is forwarded per-server for the tools preview and + * allowlist configuration, and is never written to the server row or the + * per-user credential store. The optional client credentials cover IdPs + * without dynamic client registration (e.g. a pre-registered Slack app) and + * ride the temporary authorize session only. + */ +export default function PassthroughAuthorizeSection({ + authType, + oauthFlow, +}: { + authType?: string | null; + oauthFlow: PassthroughOAuthFlow; +}) { + if (authType !== AUTH_TYPE.TRUE_PASSTHROUGH && authType !== AUTH_TYPE.OAUTH_DELEGATE) return null; + return ( +
+

+ Callers bring their own upstream token for this auth type, so LiteLLM stores no upstream credentials. To preview + tools and configure the tool allowlist, authorize against the upstream here: the token stays in this browser + session only and is never saved to LiteLLM. +

+ OAuth Client ID (optional, not saved)} + name={["credentials", "client_id"]} + extra="Only needed when the upstream does not support dynamic client registration (e.g. a pre-registered Slack app). Used for this browser authorization only." + > + + + OAuth Client Secret (optional, not saved)} + name={["credentials", "client_secret"]} + > + + + + {oauthFlow.error &&

{oauthFlow.error}

} + {oauthFlow.status === "success" && oauthFlow.tokenResponse?.access_token && ( +

+ Token held for this browser session. Tools can now be previewed and configured; nothing was saved to LiteLLM. +

+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index 27e6d6a8e3d..ca94aae20e9 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -189,6 +189,32 @@ describe("CreateMCPServer", () => { ).not.toBeInTheDocument(); }); + it.each([["True Passthrough (no LiteLLM auth)"], ["OAuth Delegate (client-supplied upstream token)"]])( + "should show the browser-only authorize section when %s is selected", + async (optionLabel) => { + await selectHttpTransport(); + + await selectAntOption("Authentication", optionLabel); + + await waitFor(() => { + expect(screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" })).toBeInTheDocument(); + }); + expect(screen.getByText("OAuth Client ID (optional, not saved)")).toBeInTheDocument(); + expect(screen.getByText("OAuth Client Secret (optional, not saved)")).toBeInTheDocument(); + }, + ); + + it("should not show the browser-only authorize section for API Key auth", async () => { + await selectHttpTransport(); + + await selectAntOption("Authentication", "API Key"); + + await waitFor(() => { + expect(screen.getByText("Authentication Value")).toBeInTheDocument(); + }); + expect(screen.queryByRole("button", { name: "Authorize & Fetch Tools (browser-only)" })).not.toBeInTheDocument(); + }); + it("should not require auth value when creating a server with API Key auth type", async () => { await selectHttpTransport(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 1362835f475..b123e3397a1 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -17,6 +17,7 @@ import { } from "./types"; import OAuthFormFields from "./OAuthFormFields"; import TruePassthroughWarning from "./TruePassthroughWarning"; +import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection"; import TokenExchangeFormFields from "./TokenExchangeFormFields"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPConnectionStatus from "./mcp_connection_status"; @@ -980,6 +981,16 @@ const CreateMCPServer: React.FC = ({ + + {shouldShowAuthValueField && ( { expect(mockGetToken).toHaveBeenCalledWith("oauth_server_1", "user-1"); }); + it("forwards the sessionStorage token as the x-mcp header for an oauth_delegate server", async () => { + mockIsTokenValid.mockReturnValue(true); + mockGetToken.mockReturnValue({ access_token: "browser-token" }); + + render( + , + ); + + await waitFor(() => { + expect(networking.listMCPTools).toHaveBeenCalledWith( + "access-token", + "oauth_server_1", + { "x-mcp-oauth_server-authorization": "Bearer browser-token" }, + true, + ); + }); + expect(mockGetToken).toHaveBeenCalledWith("oauth_server_1", "user-1"); + }); + + it("prompts for the browser-only authorize when a true_passthrough server has no token", async () => { + mockIsTokenValid.mockReturnValue(false); + + render( + , + ); + + await waitFor(() => { + expect(screen.getByTestId("mcp-tool-config").getAttribute("data-external-error")).toContain( + "Authorize with the upstream (browser-only", + ); + }); + expect(networking.listMCPTools).not.toHaveBeenCalled(); + expect(screen.getByRole("button", { name: "Authorize & Fetch Tools (browser-only)" })).toBeInTheDocument(); + }); + it("uses the staged OAuth token to load passthrough tools after authorize", async () => { const passthroughServer = { ...interactiveOAuthServer, delegate_auth_to_upstream: true }; mockIsTokenValid.mockReturnValue(false); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 2c4f674c14b..ee67ead8121 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -19,6 +19,7 @@ import { buildMcpPassthroughAuthHeader } from "@/utils/mcpHeaderUtils"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPPermissionManagement from "./MCPPermissionManagement"; import TruePassthroughWarning from "./TruePassthroughWarning"; +import PassthroughAuthorizeSection from "./PassthroughAuthorizeSection"; import MCPToolConfiguration from "./mcp_tool_configuration"; import StdioConfiguration from "./StdioConfiguration"; import TokenExchangeFormFields from "./TokenExchangeFormFields"; @@ -172,20 +173,40 @@ const MCPServerEdit: React.FC = ({ }; }, onTokenReceived: (token) => { - if (token?.access_token) { - const credentials = { - access_token: token.access_token, - ...(token.refresh_token && { refresh_token: token.refresh_token }), - ...(token.expires_in && { expires_in: token.expires_in }), - ...(token.scope && { scope: token.scope }), - }; - - form.setFieldsValue({ credentials }); - - NotificationsManager.success( - "OAuth authorization successful! Please click 'Update MCP Server' to save the credentials.", - ); + if (!token?.access_token) { + return; } + + const effectiveAuthType = form.getFieldValue("auth_type") ?? mcpServer.auth_type; + if (effectiveAuthType === AUTH_TYPE.TRUE_PASSTHROUGH || effectiveAuthType === AUTH_TYPE.OAUTH_DELEGATE) { + setToken( + mcpServer.server_id, + { + access_token: token.access_token, + expires_in: token.expires_in, + refresh_token: token.refresh_token, + token_type: token.token_type, + }, + userID, + ); + NotificationsManager.success( + "Token held for this browser session. Tools can now be loaded and configured; nothing was saved to LiteLLM.", + ); + return; + } + + const credentials = { + access_token: token.access_token, + ...(token.refresh_token && { refresh_token: token.refresh_token }), + ...(token.expires_in && { expires_in: token.expires_in }), + ...(token.scope && { scope: token.scope }), + }; + + form.setFieldsValue({ credentials }); + + NotificationsManager.success( + "OAuth authorization successful! Please click 'Update MCP Server' to save the credentials.", + ); }, onBeforeRedirect: persistEditUiState, flowSource: "edit", @@ -369,7 +390,9 @@ const MCPServerEdit: React.FC = ({ oauth2_flow: mcpServer.oauth2_flow, delegate_auth_to_upstream: mcpServer.delegate_auth_to_upstream, }) === "passthrough"; - if (isPassthrough) { + const isBrowserHeldTokenMode = + mcpServer.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || mcpServer.auth_type === AUTH_TYPE.OAUTH_DELEGATE; + if (isPassthrough || isBrowserHeldTokenMode) { const token = oauthTokenResponse?.access_token ?? (isTokenValid(mcpServer.server_id, userID) @@ -377,7 +400,11 @@ const MCPServerEdit: React.FC = ({ : null); if (!token) { setTools([]); - setToolsError("Authenticate with this server in the Tools tab to load and configure its tools."); + setToolsError( + isBrowserHeldTokenMode + ? "Authorize with the upstream (browser-only, in the Authentication section) to load and configure this server's tools." + : "Authenticate with this server in the Tools tab to load and configure its tools.", + ); return; } customHeaders = buildMcpPassthroughAuthHeader(mcpServer.alias, token); @@ -893,6 +920,15 @@ const MCPServerEdit: React.FC = ({ + )} diff --git a/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx b/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx index 055e15350ca..3208b6b02b2 100644 --- a/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx +++ b/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx @@ -56,7 +56,9 @@ export const useTestMCPConnection = ({ // Check if we have the minimum required fields to fetch tools const isM2MOAuth = formValues.auth_type === AUTH_TYPE.OAUTH2 && formValues.oauth_flow_type === OAUTH_FLOW.M2M; - const requiresOAuthToken = formValues.auth_type === AUTH_TYPE.OAUTH2 && !isM2MOAuth; + const isBrowserHeldTokenMode = + formValues.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || formValues.auth_type === AUTH_TYPE.OAUTH_DELEGATE; + const requiresOAuthToken = (formValues.auth_type === AUTH_TYPE.OAUTH2 && !isM2MOAuth) || isBrowserHeldTokenMode; const isOpenAPITransport = formValues.transport === TRANSPORT.OPENAPI; const hasEndpoint = isOpenAPITransport ? !!formValues.spec_path : !!formValues.url; From 367aa904de68c6945c6006261b6a1ff6017b750f Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 7 Jul 2026 21:52:56 -0700 Subject: [PATCH 051/101] fix(ui): wire the tool playground and gateway authorize flow for the client-forwarded token modes The server detail page's Tool Testing Playground gated its browser-held token handling on the legacy PKCE-passthrough shape, so a true_passthrough or oauth_delegate server listed tools unauthenticated and surfaced 'Failed to fetch MCP tools' with no way to authorize. The playground now treats both modes as browser-held-token servers: it reads the sessionStorage token established by the create/edit browser-only Authorize, forwards it via the x-mcp-{alias}-authorization header, evicts it on a 401, and shows its own Authorize gate when the token is absent. That gate's flow uses the gateway's relayed authorize/register/token endpoints with the real server id, which previously 400ed for anything but oauth2. Those endpoints now also accept the client-forwarded token modes (the minted token is upstream-audienced and browser-held; DCR persistence stays off on this path), and registry builds run the same RFC 9728/8414 endpoint discovery for these modes that oauth2 rows get, since their rows never store an authorization_url. --- .../mcp_server/discoverable_endpoints.py | 14 +++-- .../mcp_server/mcp_server_manager.py | 5 +- .../mcp_server/test_discoverable_endpoints.py | 55 +++++++++++++++++++ .../mcp_server/test_mcp_server_manager.py | 33 +++++++++++ .../components/mcp_tools/mcp_tools.test.tsx | 31 +++++++++++ .../src/components/mcp_tools/mcp_tools.tsx | 23 +++++--- 6 files changed, 147 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index c87e900aa2a..7606deac241 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -465,8 +465,15 @@ async def _store_per_user_token_server_side( def _raise_if_not_oauth2(mcp_server: MCPServer) -> None: - """Reject a non-oauth2 server from the gateway's OAuth authorize/token/register flow.""" - if mcp_server.auth_type == MCPAuth.oauth2: + """Reject a server without upstream OAuth from the gateway's authorize/token/register flow. + + The client-forwarded token modes (``true_passthrough`` / ``oauth_delegate``) are allowed + through: the caller owns the upstream token, and this relayed flow is how a browser obtains + one against the upstream IdP (the admin UI's browser-only Authorize uses it). The minted + token is upstream-audienced and held by the caller; the gateway persists nothing for these + modes (DCR persistence is opt-in and never enabled on this path). + """ + if mcp_server.auth_type in (MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate): return raise HTTPException( status_code=400, @@ -515,8 +522,7 @@ async def authorize_with_server( response_type: Optional[str] = None, scope: Optional[str] = None, ): - if mcp_server.auth_type != "oauth2": - raise HTTPException(status_code=400, detail="MCP server is not OAuth2") + _raise_if_not_oauth2(mcp_server) if mcp_server.authorization_url is None: raise HTTPException(status_code=400, detail="MCP server authorization url is not set") diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index c4ad673b88f..c8681b94f5e 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1384,8 +1384,9 @@ class MCPServerManager: auth_type = cast(MCPAuthType, mcp_server.auth_type) server_url = mcp_server.url + upstream_oauth_auth_types = (MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate) needs_discovery = bool(server_url) and ( - (auth_type == MCPAuth.oauth2 and not mcp_server.authorization_url) + (auth_type in upstream_oauth_auth_types and not mcp_server.authorization_url) or self._obo_needs_endpoint_discovery( auth_type, mcp_server.token_exchange_endpoint @@ -1396,7 +1397,7 @@ class MCPServerManager: mcp_oauth_metadata = ( await self._descovery_metadata( server_url=server_url, # type: ignore[arg-type] - allow_origin_fallback=auth_type == MCPAuth.oauth2, + allow_origin_fallback=auth_type in upstream_oauth_auth_types, ) if needs_discovery else None diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 19d030f17c4..926c3d5a1f4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -119,6 +119,61 @@ async def test_authorize_endpoint_includes_response_type(): assert "scope=read+write" in response.headers["location"] +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_type_value", ["true_passthrough", "oauth_delegate"]) +async def test_authorize_endpoint_allows_client_forwarded_modes(auth_type_value): + """The browser-only Authorize relays the gateway authorize flow for the client-forwarded + token modes; the oauth2-only gate must let them through and redirect to the upstream IdP.""" + try: + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + authorize, + ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + except ImportError: + pytest.skip("MCP discoverable endpoints not available") + + global_mcp_server_manager.registry.clear() + + server = MCPServer( + server_id="test_cf_server", + name="test_cf", + server_name="test_cf", + alias="test_cf", + transport=MCPTransport.http, + auth_type=MCPAuth(auth_type_value), + # Discovery stamps these onto the in-memory registry entry at build time. + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + ) + global_mcp_server_manager.registry[server.server_id] = server + + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + with patch("litellm.proxy._experimental.mcp_server.discoverable_endpoints.encrypt_value_helper") as mock_encrypt: + mock_encrypt.return_value = "mocked_encrypted_state" + + response = await authorize( + request=mock_request, + client_id="dcr_client_id", + mcp_server_name="test_cf", + redirect_uri="http://127.0.0.1:60108/callback", + state="test_state", + ) + + assert response.status_code == 307 + assert "https://provider.com/oauth/authorize" in response.headers["location"] + assert "client_id=dcr_client_id" in response.headers["location"] + + @pytest.mark.asyncio async def test_authorize_endpoint_preserves_existing_query_params(): """Test that authorize endpoint merges OAuth params with existing query params in authorization_url""" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 12e22de195b..e8fca9ac6ab 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -833,6 +833,39 @@ class TestMCPServerManager: assert spec is not None and isinstance(spec.config, TokenExchangeConfig) assert spec.config.profile == "entra_obo" + @pytest.mark.asyncio + @pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) + async def test_build_from_table_discovers_upstream_oauth_for_client_forwarded_modes(self, auth_type): + """The gateway's relayed authorize flow (used by the browser-only Authorize) needs the + upstream's authorization_url on the registry entry, and these rows never persist one, so + the DB build must discover it the same way oauth2 rows do.""" + from types import SimpleNamespace + + manager = MCPServerManager() + row = LiteLLM_MCPServerTable( + server_id="cf-db-1", + alias="cf_db", + description="client-forwarded from db", + url="https://up.example.com/mcp", + transport=MCPTransport.http, + auth_type=auth_type, + created_at=datetime.now(), + updated_at=datetime.now(), + ) + + metadata = SimpleNamespace( + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + registration_url="https://idp.example.com/register", + scopes=None, + ) + with patch.object(manager, "_descovery_metadata", new=AsyncMock(return_value=metadata)) as mock_discovery: + built = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + + mock_discovery.assert_awaited_once() + assert built.authorization_url == "https://idp.example.com/authorize" + assert built.token_url == "https://idp.example.com/token" + async def _capture_subject_token(self, call) -> Optional[str]: """Run a manager method (via ``call(manager)``) and return the subject_token it threaded into ``_create_mcp_client``.""" diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.test.tsx index 3346bc342f3..2e8fa901f6d 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.test.tsx @@ -91,6 +91,37 @@ describe("MCPToolsViewer auth gate routing", () => { expect(screen.queryByText(GATE_TEXT)).not.toBeInTheDocument(); }); + it.each([["true_passthrough"], ["oauth_delegate"]])( + "shows the Authorize gate for a %s server without a browser token and does not list tools", + async (authType) => { + renderViewer({ auth_type: authType, oauth2_flow: null, delegate_auth_to_upstream: false }); + + expect(await screen.findByText(GATE_TEXT)).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Authorize" })).toBeInTheDocument(); + expect(vi.mocked(listMCPTools)).not.toHaveBeenCalled(); + expect(vi.mocked(getMCPOAuthUserCredentialStatus)).not.toHaveBeenCalled(); + }, + ); + + it.each([["true_passthrough"], ["oauth_delegate"]])( + "forwards the session token via the x-mcp header for a %s server that has one", + async (authType) => { + vi.mocked(isTokenValid).mockReturnValue(true); + vi.mocked(getToken).mockReturnValue({ access_token: "upstream-tok" } as ReturnType); + + renderViewer({ auth_type: authType, oauth2_flow: null, delegate_auth_to_upstream: false }); + + await waitFor(() => + expect(vi.mocked(listMCPTools)).toHaveBeenCalledWith( + "litellm-key", + "srv-1", + expect.objectContaining({ "x-mcp-slack-authorization": "Bearer upstream-tok" }), + ), + ); + expect(screen.queryByText(GATE_TEXT)).not.toBeInTheDocument(); + }, + ); + it("lists tools for an OBO server when the user has a DB credential, with no x-mcp header", async () => { renderViewer({ oauth2_flow: null, delegate_auth_to_upstream: false }); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx index e436732ba3b..6c99a53afaa 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx @@ -2,7 +2,7 @@ import React, { useCallback, useEffect, useState } from "react"; import { useQuery, useMutation } from "@tanstack/react-query"; import { ToolTestPanel } from "./ToolTestPanel"; import { resolveLogoSrc } from "@/lib/assetPaths"; -import { MCPTool, MCPToolsViewerProps, MCPContent, CallMCPToolResponse, getMcpOAuthMode } from "./types"; +import { AUTH_TYPE, MCPTool, MCPToolsViewerProps, MCPContent, CallMCPToolResponse, getMcpOAuthMode } from "./types"; import { listMCPTools, callMCPTool, getMCPOAuthUserCredentialStatus } from "../networking"; import { isTokenValid, getToken, removeToken } from "@/utils/mcpTokenStore"; import { sanitizeMcpAliasForHeader, buildMcpPassthroughAuthHeader } from "@/utils/mcpHeaderUtils"; @@ -42,19 +42,24 @@ const MCPToolsViewer = ({ // service token and needs no gate. const oauthMode = getMcpOAuthMode({ auth_type, oauth2_flow, delegate_auth_to_upstream }); const isPassthrough = oauthMode === "passthrough"; + // The client-forwarded token modes gate the same way as PKCE passthrough: the + // browser session token (established via the browser-only Authorize in the + // create/edit forms, or right here) is the upstream credential. + const usesBrowserHeldToken = + isPassthrough || auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || auth_type === AUTH_TYPE.OAUTH_DELEGATE; const isAuthorizationCode = oauthMode === "authorization_code"; const [oauthToken, setOauthToken] = useState(() => - isPassthrough && isTokenValid(serverId, userID) ? getToken(serverId, userID)?.access_token ?? null : null, + usesBrowserHeldToken && isTokenValid(serverId, userID) ? getToken(serverId, userID)?.access_token ?? null : null, ); // Re-sync token when serverId/userID changes (useState initializer only runs on mount). useEffect(() => { - if (!isPassthrough) { + if (!usesBrowserHeldToken) { setOauthToken(null); return; } setOauthToken(isTokenValid(serverId, userID) ? getToken(serverId, userID)?.access_token ?? null : null); - }, [serverId, userID, isPassthrough]); + }, [serverId, userID, usesBrowserHeldToken]); const { startOAuthFlow, @@ -109,7 +114,7 @@ const MCPToolsViewer = ({ // x-mcp-{alias}-{header} pattern and forwards it to the upstream MCP server. // When no alias is available, fall back to x-mcp-auth (legacy but still supported). // Passthrough only: authorization_code/token_exchange/M2M tokens are attached server-side, not from the browser. - if (isPassthrough && oauthToken) { + if (usesBrowserHeldToken && oauthToken) { Object.assign(customHeaders, buildMcpPassthroughAuthHeader(serverAlias, oauthToken)); } @@ -164,7 +169,8 @@ const MCPToolsViewer = ({ // Passthrough blocks until a browser session token exists; authorization_code blocks until // the user has a valid DB credential (else the backend returns no tools). enabled: - !!accessToken && (isPassthrough ? oauthToken !== null : isAuthorizationCode ? hasAuthorizationCodeCred : true), + !!accessToken && + (usesBrowserHeldToken ? oauthToken !== null : isAuthorizationCode ? hasAuthorizationCodeCred : true), staleTime: 30000, // Consider data fresh for 30 seconds retry: (failureCount, error: any) => { // Don't retry on 401 — token is invalid, user must re-authenticate @@ -253,7 +259,8 @@ const MCPToolsViewer = ({ // passthrough needs a browser token; authorization_code needs a stored DB credential or a // still-valid one — a 401 from the list call means the backend has none even // after attempting a refresh, so re-authorization is required. - const authGateActive = (isPassthrough && !oauthToken) || authorizationCodeNeedsAuth || authorizationCodeTokenRejected; + const authGateActive = + (usesBrowserHeldToken && !oauthToken) || authorizationCodeNeedsAuth || authorizationCodeTokenRejected; // Treat authorization_code credential-status loading as "tools loading" so the empty state // doesn't flash before we know whether the user needs to authorize. const toolsAreaLoading = isLoadingTools || authorizationCodeStatusLoading; @@ -359,7 +366,7 @@ const MCPToolsViewer = ({ {/* Passthrough auth gate — browser session token absent */} - {isPassthrough && !oauthToken && ( + {usesBrowserHeldToken && !oauthToken && (

Authentication required

From 74a15c21ae2324af75f786b748aff9313e609f8e Mon Sep 17 00:00:00 2001 From: Tin Date: Tue, 7 Jul 2026 22:45:48 -0700 Subject: [PATCH 052/101] fix(mcp): run upstream OAuth endpoint discovery for config-defined client-forwarded servers The DB build already discovers authorization/token endpoints for true_passthrough and oauth_delegate rows; the config.yaml load path kept the oauth2-only gate, so a YAML-defined server in either mode could not use the relayed authorize flow unless the YAML declared authorization_url. Both paths now share the same auth type set. --- litellm/proxy/_experimental/mcp_server/mcp_server_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index c8681b94f5e..dd7d3e96262 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -983,8 +983,9 @@ class MCPServerManager: ) auth_type = server_config.get("auth_type", None) + config_upstream_oauth_auth_types = (MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate) if server_url and ( - auth_type == MCPAuth.oauth2 + auth_type in config_upstream_oauth_auth_types or self._obo_needs_endpoint_discovery( auth_type, server_config.get("token_exchange_endpoint"), @@ -993,7 +994,7 @@ class MCPServerManager: ): mcp_oauth_metadata = await self._descovery_metadata( server_url=server_url, - allow_origin_fallback=auth_type == MCPAuth.oauth2, + allow_origin_fallback=auth_type in config_upstream_oauth_auth_types, ) else: mcp_oauth_metadata = None From 98818df418f4e613e510500c91ad1a86a12c1d6c Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 14:28:19 -0700 Subject: [PATCH 053/101] fix(mcp): recognize per-server auth header at connect and stop persisting browser-authorize tokens Two correctness fixes for the client-forwarded token modes. The preemptive-401 connect gate for true_passthrough and oauth_delegate only inspected the request-wide Authorization, so a caller who bound the upstream token via the per-server x-mcp-{alias}-authorization header (the mandatory shape in a multi-server aggregate, where the request-wide Authorization is withheld) was spuriously 401'd at connect even though egress already honors that header. The gate now recognizes the per-server header for both modes via a shared helper, mode-correctly: true_passthrough treats any Authorization or the per-server header as the upstream token, oauth_delegate keeps requiring a distinct x-litellm-api-key so a lone Authorization consumed for admission is never mistaken for an upstream token. The preemptive raise is also gated to single-server scopes so a multi-server aggregate degrades gracefully (the listing absorbs a per-server failure) instead of one missing token 401-ing the whole connect. The browser-only Authorize flow was writing the upstream access and refresh token to LiteLLM_MCPUserCredentials, contradicting the modes' persist-nothing contract: the temp OAuth-relay server was cached with a hardcoded oauth2 auth_type, so needs_user_oauth_token was true and the token exchange stored it. The create and edit forms now send the real auth_type for these modes, so the temp server is not oauth2, needs_user_oauth_token is false, and the exchange skips storage while still returning the token to the browser session. --- .../mcp_server/test_discoverable_endpoints.py | 79 +++++++++++++++++++ .../mcp_tools/create_mcp_server.tsx | 5 +- .../components/mcp_tools/mcp_server_edit.tsx | 5 +- 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 926c3d5a1f4..9de342eabd7 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -6,6 +6,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException +from litellm.types.mcp import MCPAuth + # Fixture to mock IP address check for all MCP tests # This prevents tests from failing due to IP-based access control @@ -3403,6 +3405,83 @@ async def test_token_exchange_passes_through_upstream_expires_in(): assert body["expires_in"] == 43200 +async def _exchange_persistence_attempted_for_auth_type(auth_type) -> bool: + """Run exchange_token_with_server for a server of ``auth_type`` and report whether it attempted + to persist the exchanged token server-side. The client-forwarded token modes must not persist: + their contract is that the upstream token stays browser-held, minted/stored/refreshed nowhere.""" + from fastapi import Request + + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( + exchange_token_with_server, + ) + from litellm.proxy._types import MCPTransport + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + server = MCPServer( + server_id="t", + name="t", + server_name="t", + alias="t", + transport=MCPTransport.http, + auth_type=auth_type, + client_id="cid", + client_secret="cs", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + ) + mock_request = MagicMock(spec=Request) + mock_request.base_url = "https://litellm.example.com/" + mock_request.headers = {} + + fake_http_response = MagicMock() + fake_http_response.json.return_value = {"access_token": "tok", "refresh_token": "r", "token_type": "Bearer"} + fake_http_response.raise_for_status = MagicMock() + fake_http_client = MagicMock() + fake_http_client.post = AsyncMock(return_value=fake_http_response) + + with ( + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints.get_async_httpx_client", + return_value=fake_http_client, + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._extract_user_id_from_request", + new_callable=AsyncMock, + return_value="admin-user", + ), + patch( + "litellm.proxy._experimental.mcp_server.discoverable_endpoints._store_per_user_token_server_side", + new_callable=AsyncMock, + ) as mock_store, + ): + await exchange_token_with_server( + request=mock_request, + mcp_server=server, + grant_type="authorization_code", + code="c", + redirect_uri="http://127.0.0.1:3000/cb", + client_id="cid", + client_secret=None, + code_verifier=None, + ) + return mock_store.await_count > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("auth_type", [MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) +async def test_token_exchange_does_not_persist_for_client_forwarded_modes(auth_type): + """The browser-only Authorize for true_passthrough / oauth_delegate must not write the upstream + token to the DB: these modes forward a browser-held token and persist nothing server-side.""" + assert await _exchange_persistence_attempted_for_auth_type(auth_type) is False + + +@pytest.mark.asyncio +async def test_token_exchange_persists_for_oauth2(): + """Guard the test's own discriminator: a genuine oauth2 (authorization_code) server DOES persist, + so the passthrough no-persist assertion above is meaningful and not vacuously true.""" + assert await _exchange_persistence_attempted_for_auth_type(MCPAuth.oauth2) is True + + # ------------------------------------------------------------------- # OBO (token_exchange) Protected Resource Metadata: discovery must name the # JWT-auth issuer the client SSOs with, not the gateway. diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index b123e3397a1..ad7d4530c92 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -184,7 +184,10 @@ const CreateMCPServer: React.FC = ({ description: values.description, url, transport: transport === TRANSPORT.OPENAPI ? "http" : transport, - auth_type: AUTH_TYPE.OAUTH2, + auth_type: + values.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || values.auth_type === AUTH_TYPE.OAUTH_DELEGATE + ? values.auth_type + : AUTH_TYPE.OAUTH2, credentials: values.credentials, authorization_url: values.authorization_url, token_url: values.token_url, diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index ee67ead8121..7d9316e1baa 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -163,7 +163,10 @@ const MCPServerEdit: React.FC = ({ description: values.description || mcpServer.description, url, transport, - auth_type: AUTH_TYPE.OAUTH2, + auth_type: + mcpServer.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || mcpServer.auth_type === AUTH_TYPE.OAUTH_DELEGATE + ? mcpServer.auth_type + : AUTH_TYPE.OAUTH2, credentials: values.credentials, mcp_access_groups: values.mcp_access_groups || mcpServer.mcp_access_groups, static_headers: staticHeaders, From 1693761a51ef0e6481a784088d44f901fbc1cc2c Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 14:34:20 -0700 Subject: [PATCH 054/101] feat(mcp): record auth_mode and upstream resource on MCP tool-call logs Adds mcp_auth_mode and mcp_server_resource to StandardLoggingMCPToolCall so a relayed passthrough/delegate request can be attributed in an audit to its mode and its upstream target without logging any credential. Both are non-sensitive metadata derived from the resolved server; the admission and upstream tokens stay SecretStr and are never logged. --- litellm/proxy/_experimental/mcp_server/server.py | 2 ++ litellm/types/utils.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 55a0fa083c0..5c814774fda 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -3064,6 +3064,8 @@ if MCP_AVAILABLE: mcp_server_logo_url=mcp_info.get("logo_url"), namespaced_tool_name=namespaced_tool_name, mcp_session_id=session_id, + mcp_auth_mode=mcp_server.auth_type, + mcp_server_resource=mcp_server.url, ) else: return StandardLoggingMCPToolCall( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 908f5b76424..e71c42084d1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2523,6 +2523,20 @@ class StandardLoggingMCPToolCall(TypedDict, total=False): the client is driving a stateful session. Absent for stateless calls. """ + mcp_auth_mode: Optional[str] + """ + The server's auth_type for this call (e.g. `true_passthrough`, `oauth_delegate`, + `oauth2`). For the client-forwarded token modes this records that the caller's own + upstream token was relayed, so an audit can attribute a relayed request to its mode + without logging any credential. + """ + + mcp_server_resource: Optional[str] + """ + The upstream MCP server URL (the RFC 8707 resource) the tool call was forwarded to. + Records which upstream received a relayed request; never a credential. + """ + class StandardLoggingVectorStoreRequest(TypedDict, total=False): """ From 7c52cde5057131469fccd1d8f6625c81c9b5d7d9 Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 15:20:07 -0700 Subject: [PATCH 055/101] fix(mcp): redact upstream URL in tool-call logs and plug fan-out Authorization bypass Two review findings on the passthrough modes. The tool-call log records the upstream MCP server URL as mcp_server_resource, which is persisted in spend-log metadata and sent to logging callbacks. A URL carrying embedded userinfo or a secret query parameter would leak into logs, so the value is now redacted to its bare resource identifier (scheme + host + path); userinfo, query string, and fragment are stripped before it is logged. The listing fan-out withholds the request-wide Authorization from a true_passthrough / oauth_delegate server when another server in scope also consumes it, so one bearer is not replayed across upstreams. The later server.extra_headers copy loop did not honor that decision: a server listing Authorization in extra_headers would re-copy the withheld bearer from raw_headers. The withhold decision is now computed once and applied to both the forwarding branch and the extra_headers loop. --- .../proxy/_experimental/mcp_server/server.py | 23 ++++++++++++++++++- litellm/types/utils.py | 4 +++- .../mcp_server/test_mcp_server.py | 22 ++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 5c814774fda..938cc2bc43b 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -27,6 +27,7 @@ from typing import ( Union, cast, ) +from urllib.parse import urlsplit, urlunsplit import httpx from fastapi import FastAPI, HTTPException @@ -105,6 +106,26 @@ _MAX_STATEFUL_SESSIONS_PER_OWNER = 100 _MCP_ROUTING_PEEK_MAX_BYTES = 4096 +def _redact_mcp_resource_url(url: Optional[str]) -> Optional[str]: + """Reduce an MCP server URL to its bare resource identifier for logging. + + Keeps scheme, host, and path; drops userinfo (``user:pass@``), the query string, + and the fragment, so an upstream URL carrying an embedded token, userinfo, or a + secret query parameter never reaches spend-log metadata or logging callbacks. + Returns None when the URL has no host to identify (nothing safe to log). + """ + if not isinstance(url, str) or not url: + return None + try: + parts = urlsplit(url) + except ValueError: + return None + if not parts.hostname: + return None + netloc = f"{parts.hostname}:{parts.port}" if parts.port else parts.hostname + return urlunsplit((parts.scheme, netloc, parts.path, "", "")) or None + + def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None: """Remove a (user_id, server_id) entry from the BYOK credential cache. @@ -3065,7 +3086,7 @@ if MCP_AVAILABLE: namespaced_tool_name=namespaced_tool_name, mcp_session_id=session_id, mcp_auth_mode=mcp_server.auth_type, - mcp_server_resource=mcp_server.url, + mcp_server_resource=_redact_mcp_resource_url(mcp_server.url), ) else: return StandardLoggingMCPToolCall( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e71c42084d1..6b99cfa3314 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2533,7 +2533,9 @@ class StandardLoggingMCPToolCall(TypedDict, total=False): mcp_server_resource: Optional[str] """ - The upstream MCP server URL (the RFC 8707 resource) the tool call was forwarded to. + The upstream MCP server resource identifier (scheme + host + path) the tool call was + forwarded to. Redacted for logging: userinfo, query string, and fragment are stripped so an + upstream URL carrying an embedded token or secret query parameter never reaches log metadata. Records which upstream received a relayed request; never a credential. """ diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index e9dccdcd4ad..0db36e75e48 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -6854,3 +6854,25 @@ async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow(): resolved = captured_servers["allowed"] assert resolved and resolved[0].oauth2_flow == "client_credentials" assert resolved[0].has_client_credentials is True + + +@pytest.mark.parametrize( + "url, expected", + [ + # userinfo + secret query param must both be stripped from the logged resource + ("https://user:s3cr3t@mcp.example.com/mcp?token=abcd1234&x=1", "https://mcp.example.com/mcp"), + ("https://mcp.example.com/mcp#frag", "https://mcp.example.com/mcp"), + ("https://host:8443/a/b?q=1", "https://host:8443/a/b"), + ("https://mcp.notion.com/mcp", "https://mcp.notion.com/mcp"), + (None, None), + ("", None), + ("not a url", None), + ], +) +def test_redact_mcp_resource_url_strips_credentials(url, expected): + """The MCP tool-call log records the upstream resource, so the URL must be redacted to + scheme+host+path: userinfo, query string, and fragment (which can carry embedded tokens or + secret parameters) must never reach spend-log metadata or logging callbacks.""" + from litellm.proxy._experimental.mcp_server.server import _redact_mcp_resource_url + + assert _redact_mcp_resource_url(url) == expected From b62b30bac0d575425da7e484cbd4f93360847b6a Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 16:10:30 -0700 Subject: [PATCH 056/101] fix(ui): edit-form browser-authorize payload uses the selected auth_type The edit form's getTemporaryPayload read the server's stored auth_type instead of the value the admin selected in the dropdown, so an admin who switched an existing oauth2 server to true_passthrough (or oauth_delegate) and ran the browser authorize flow built the temporary OAuth-relay server as oauth2. That made needs_user_oauth_token true and persisted the token to the DB, contrary to the mode's browser-held contract, and left it inconsistent with onTokenReceived and the submit payload, both of which already read the form value. It now reads values.auth_type, matching the create form. --- .../mcp_tools/mcp_server_edit.test.tsx | 44 ++++++++++++++++--- .../components/mcp_tools/mcp_server_edit.tsx | 4 +- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index f071e247466..0579a3208d1 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -19,14 +19,20 @@ vi.mock("../molecules/notifications_manager", () => ({ }, })); -const mockOauth: { tokenResponse: any } = { tokenResponse: null }; +const mockOauth: { + tokenResponse: any; + getTemporaryPayload: (() => Record | null) | null; +} = { tokenResponse: null, getTemporaryPayload: null }; vi.mock("@/hooks/useMcpOAuthFlow", () => ({ - useMcpOAuthFlow: () => ({ - startOAuthFlow: vi.fn(), - status: "idle", - error: null, - tokenResponse: mockOauth.tokenResponse, - }), + useMcpOAuthFlow: (opts: { getTemporaryPayload?: () => Record | null }) => { + mockOauth.getTemporaryPayload = opts?.getTemporaryPayload ?? null; + return { + startOAuthFlow: vi.fn(), + status: "idle", + error: null, + tokenResponse: mockOauth.tokenResponse, + }; + }, })); vi.mock("./mcp_server_cost_config", () => ({ @@ -344,6 +350,30 @@ describe("MCPServerEdit (true passthrough warning)", () => { screen.queryByText("True Passthrough disables LiteLLM authentication for this server"), ).not.toBeInTheDocument(); }); + + it("browser-authorize temp payload uses the selected auth_type, not the stored one", async () => { + // Stored server is oauth2; the admin switches the dropdown to true_passthrough before saving. + // The temp OAuth-relay payload must reflect the selection so the exchange is treated as + // browser-held (no DB persistence), matching onTokenReceived and the create form. + render( + , + ); + + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + await waitFor(() => { + expect(mockOauth.getTemporaryPayload).toBeTruthy(); + }); + const payload = mockOauth.getTemporaryPayload!(); + expect(payload).toBeTruthy(); + expect(payload?.auth_type).toBe("true_passthrough"); + }); }); describe("MCPServerEdit (auth type switch)", () => { diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 7d9316e1baa..0a00f05ab53 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -164,8 +164,8 @@ const MCPServerEdit: React.FC = ({ url, transport, auth_type: - mcpServer.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || mcpServer.auth_type === AUTH_TYPE.OAUTH_DELEGATE - ? mcpServer.auth_type + values.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || values.auth_type === AUTH_TYPE.OAUTH_DELEGATE + ? values.auth_type : AUTH_TYPE.OAUTH2, credentials: values.credentials, mcp_access_groups: values.mcp_access_groups || mcpServer.mcp_access_groups, From ee5a0651161b8c71a5852e2590e6c9f9285f937b Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 17:39:01 -0700 Subject: [PATCH 057/101] refactor(ui): extract isClientForwardedTokenMode helper for the pass-through modes The 'auth_type is true_passthrough or oauth_delegate' check was duplicated inline across both server forms' browser-authorize temp payloads, the edit form's onTokenReceived and tool-preview gate, PassthroughAuthorizeSection, and mcp_tools' usesBrowserHeldToken. Extracted a single isClientForwardedTokenMode helper in types.tsx and routed every site through it so the set of client-forwarded modes lives in one place and cannot drift. Also replaced a pre-existing nested ternary in the authorize button label surfaced by touching the file. --- ui/litellm-dashboard/eslint-metrics.json | 2 +- .../mcp_tools/PassthroughAuthorizeSection.tsx | 15 ++++++++------- .../components/mcp_tools/create_mcp_server.tsx | 6 ++---- .../src/components/mcp_tools/mcp_server_edit.tsx | 11 ++++------- .../src/components/mcp_tools/mcp_tools.tsx | 12 +++++++++--- .../src/components/mcp_tools/types.tsx | 7 +++++++ 6 files changed, 31 insertions(+), 22 deletions(-) diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 37bad071081..e4647b33d61 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,7 +1,7 @@ { "@typescript-eslint/no-explicit-any": 1982, "complexity": 128, - "local/no-large-inline-object-arg": 519, + "local/no-large-inline-object-arg": 513, "local/no-long-condition-chain": 233, "max-depth": 59, "no-console": 15 diff --git a/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx index 453e3024000..af81f2713ae 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/PassthroughAuthorizeSection.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Button, Form, Input } from "antd"; -import { AUTH_TYPE } from "./types"; +import { isClientForwardedTokenMode } from "./types"; interface PassthroughOAuthFlow { startOAuthFlow: () => void | Promise; @@ -26,7 +26,12 @@ export default function PassthroughAuthorizeSection({ authType?: string | null; oauthFlow: PassthroughOAuthFlow; }) { - if (authType !== AUTH_TYPE.TRUE_PASSTHROUGH && authType !== AUTH_TYPE.OAUTH_DELEGATE) return null; + if (!isClientForwardedTokenMode(authType)) return null; + const authorizeButtonLabels: Record = { + authorizing: "Waiting for authorization...", + exchanging: "Exchanging authorization code...", + }; + const authorizeButtonLabel = authorizeButtonLabels[oauthFlow.status] ?? "Authorize & Fetch Tools (browser-only)"; return (

@@ -57,11 +62,7 @@ export default function PassthroughAuthorizeSection({ onClick={oauthFlow.startOAuthFlow} disabled={oauthFlow.status === "authorizing" || oauthFlow.status === "exchanging"} > - {oauthFlow.status === "authorizing" - ? "Waiting for authorization..." - : oauthFlow.status === "exchanging" - ? "Exchanging authorization code..." - : "Authorize & Fetch Tools (browser-only)"} + {authorizeButtonLabel} {oauthFlow.error &&

{oauthFlow.error}

} {oauthFlow.status === "success" && oauthFlow.tokenResponse?.access_token && ( diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index ad7d4530c92..7d160407d02 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -14,6 +14,7 @@ import { getMcpOAuthMode, MCP_OAUTH2_FLOW_M2M, MCP_OAUTH2_FLOW_INTERACTIVE, + isClientForwardedTokenMode, } from "./types"; import OAuthFormFields from "./OAuthFormFields"; import TruePassthroughWarning from "./TruePassthroughWarning"; @@ -184,10 +185,7 @@ const CreateMCPServer: React.FC = ({ description: values.description, url, transport: transport === TRANSPORT.OPENAPI ? "http" : transport, - auth_type: - values.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || values.auth_type === AUTH_TYPE.OAUTH_DELEGATE - ? values.auth_type - : AUTH_TYPE.OAUTH2, + auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2, credentials: values.credentials, authorization_url: values.authorization_url, token_url: values.token_url, diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 0a00f05ab53..2c43b9cb056 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -4,6 +4,7 @@ import { InfoCircleOutlined } from "@ant-design/icons"; import { Button, TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react"; import { AUTH_TYPE, + isClientForwardedTokenMode, OAUTH_FLOW, MCP_OAUTH2_FLOW_M2M, MCP_OAUTH2_FLOW_INTERACTIVE, @@ -163,10 +164,7 @@ const MCPServerEdit: React.FC = ({ description: values.description || mcpServer.description, url, transport, - auth_type: - values.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || values.auth_type === AUTH_TYPE.OAUTH_DELEGATE - ? values.auth_type - : AUTH_TYPE.OAUTH2, + auth_type: isClientForwardedTokenMode(values.auth_type) ? values.auth_type : AUTH_TYPE.OAUTH2, credentials: values.credentials, mcp_access_groups: values.mcp_access_groups || mcpServer.mcp_access_groups, static_headers: staticHeaders, @@ -181,7 +179,7 @@ const MCPServerEdit: React.FC = ({ } const effectiveAuthType = form.getFieldValue("auth_type") ?? mcpServer.auth_type; - if (effectiveAuthType === AUTH_TYPE.TRUE_PASSTHROUGH || effectiveAuthType === AUTH_TYPE.OAUTH_DELEGATE) { + if (isClientForwardedTokenMode(effectiveAuthType)) { setToken( mcpServer.server_id, { @@ -393,8 +391,7 @@ const MCPServerEdit: React.FC = ({ oauth2_flow: mcpServer.oauth2_flow, delegate_auth_to_upstream: mcpServer.delegate_auth_to_upstream, }) === "passthrough"; - const isBrowserHeldTokenMode = - mcpServer.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || mcpServer.auth_type === AUTH_TYPE.OAUTH_DELEGATE; + const isBrowserHeldTokenMode = isClientForwardedTokenMode(mcpServer.auth_type); if (isPassthrough || isBrowserHeldTokenMode) { const token = oauthTokenResponse?.access_token ?? diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx index 6c99a53afaa..928a2e3c6bb 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_tools.tsx @@ -2,7 +2,14 @@ import React, { useCallback, useEffect, useState } from "react"; import { useQuery, useMutation } from "@tanstack/react-query"; import { ToolTestPanel } from "./ToolTestPanel"; import { resolveLogoSrc } from "@/lib/assetPaths"; -import { AUTH_TYPE, MCPTool, MCPToolsViewerProps, MCPContent, CallMCPToolResponse, getMcpOAuthMode } from "./types"; +import { + isClientForwardedTokenMode, + MCPTool, + MCPToolsViewerProps, + MCPContent, + CallMCPToolResponse, + getMcpOAuthMode, +} from "./types"; import { listMCPTools, callMCPTool, getMCPOAuthUserCredentialStatus } from "../networking"; import { isTokenValid, getToken, removeToken } from "@/utils/mcpTokenStore"; import { sanitizeMcpAliasForHeader, buildMcpPassthroughAuthHeader } from "@/utils/mcpHeaderUtils"; @@ -45,8 +52,7 @@ const MCPToolsViewer = ({ // The client-forwarded token modes gate the same way as PKCE passthrough: the // browser session token (established via the browser-only Authorize in the // create/edit forms, or right here) is the upstream credential. - const usesBrowserHeldToken = - isPassthrough || auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || auth_type === AUTH_TYPE.OAUTH_DELEGATE; + const usesBrowserHeldToken = isPassthrough || isClientForwardedTokenMode(auth_type); const isAuthorizationCode = oauthMode === "authorization_code"; const [oauthToken, setOauthToken] = useState(() => usesBrowserHeldToken && isTokenValid(serverId, userID) ? getToken(serverId, userID)?.access_token ?? null : null, diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 70cc8129bf1..dca9e574e22 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -45,6 +45,13 @@ export const AUTH_TYPE = { OAUTH_DELEGATE: "oauth_delegate", }; +// The two client-forwarded token modes: the caller supplies the upstream Authorization (forwarded +// verbatim for true_passthrough, alongside LiteLLM admission for oauth_delegate). The dashboard holds +// their token in sessionStorage instead of persisting it, and the browser-authorize temp payload keeps +// their real auth_type so the backend does not treat them as needing a stored per-user token. +export const isClientForwardedTokenMode = (authType?: string | null): boolean => + authType === AUTH_TYPE.TRUE_PASSTHROUGH || authType === AUTH_TYPE.OAUTH_DELEGATE; + export const OAUTH_FLOW = { INTERACTIVE: "interactive", M2M: "m2m", From a199bf975d588754a33331173dc4a406428a5e51 Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 17:46:28 -0700 Subject: [PATCH 058/101] refactor(mcp): share one constant for the upstream-OAuth discovery auth types The config-YAML loader and the DB loader each defined their own local tuple (oauth2, true_passthrough, oauth_delegate) to decide which auth types trigger upstream OAuth endpoint discovery, under two different names. Hoisted them to a single module constant _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES so the two load paths cannot drift on which modes get discovery. --- .../mcp_server/mcp_server_manager.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index dd7d3e96262..356ed7a2729 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -171,6 +171,16 @@ _user_env_vars_cache: dict[tuple[str, str], tuple[dict[str, str], float]] = {} _USER_ENV_VARS_CACHE_TTL = 60 # seconds _USER_ENV_VARS_CACHE_MAX_SIZE = 4096 # cap to prevent unbounded growth +# Auth types whose upstream OAuth endpoints (protected-resource + authorization-server metadata) the +# gateway discovers from the upstream itself: interactive oauth2 and the two client-forwarded modes. +# OBO/M2M endpoint discovery is decided separately via _obo_needs_endpoint_discovery. Shared by the +# config-YAML and DB server loaders so the two paths cannot drift on which modes trigger discovery. +_UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: tuple[MCPAuth, ...] = ( + MCPAuth.oauth2, + MCPAuth.true_passthrough, + MCPAuth.oauth_delegate, +) + def invalidate_user_env_vars_cache(user_id: str, server_id: str) -> None: """Drop a cached entry after the user stores or clears their env var values @@ -983,9 +993,8 @@ class MCPServerManager: ) auth_type = server_config.get("auth_type", None) - config_upstream_oauth_auth_types = (MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate) if server_url and ( - auth_type in config_upstream_oauth_auth_types + auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES or self._obo_needs_endpoint_discovery( auth_type, server_config.get("token_exchange_endpoint"), @@ -994,7 +1003,7 @@ class MCPServerManager: ): mcp_oauth_metadata = await self._descovery_metadata( server_url=server_url, - allow_origin_fallback=auth_type in config_upstream_oauth_auth_types, + allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, ) else: mcp_oauth_metadata = None @@ -1385,9 +1394,8 @@ class MCPServerManager: auth_type = cast(MCPAuthType, mcp_server.auth_type) server_url = mcp_server.url - upstream_oauth_auth_types = (MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate) needs_discovery = bool(server_url) and ( - (auth_type in upstream_oauth_auth_types and not mcp_server.authorization_url) + (auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES and not mcp_server.authorization_url) or self._obo_needs_endpoint_discovery( auth_type, mcp_server.token_exchange_endpoint @@ -1398,7 +1406,7 @@ class MCPServerManager: mcp_oauth_metadata = ( await self._descovery_metadata( server_url=server_url, # type: ignore[arg-type] - allow_origin_fallback=auth_type in upstream_oauth_auth_types, + allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, ) if needs_discovery else None From e29e24e628661e1faf75955e66d185386ddf5a4d Mon Sep 17 00:00:00 2001 From: Tin Date: Wed, 8 Jul 2026 23:09:19 -0700 Subject: [PATCH 059/101] fix(ui): keep the browser-authorized token out of form.credentials for the pass-through modes The create form wrote the upstream token obtained by Authorize & Fetch into form.credentials for every mode, so for true_passthrough / oauth_delegate the browser-held token leaked into the OAuth flow's getCredentials (preview requests) and the redirect-persist cache, and was a step away from server-level credential persistence. onTokenReceived now early-returns for the client-forwarded modes, holding the token only in local state for preview (mirroring the edit form), instead of writing it into form.credentials. --- .../mcp_tools/create_mcp_server.test.tsx | 25 +++++++++++ .../mcp_tools/create_mcp_server.tsx | 45 ++++++++++++------- 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index ca94aae20e9..eecbc253b6b 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -30,6 +30,7 @@ const oauthHook = vi.hoisted(() => ({ onTokenReceived: null as | ((token: Record | null, registeredClient?: { clientId?: string; clientSecret?: string }) => void) | null, + getCredentials: null as (() => Record | undefined) | null, })); vi.mock("@/hooks/useMcpOAuthFlow", () => ({ useMcpOAuthFlow: (opts: { @@ -37,8 +38,10 @@ vi.mock("@/hooks/useMcpOAuthFlow", () => ({ token: Record | null, registeredClient?: { clientId?: string; clientSecret?: string }, ) => void; + getCredentials?: () => Record | undefined; }) => { oauthHook.onTokenReceived = opts.onTokenReceived; + oauthHook.getCredentials = opts.getCredentials ?? null; return { startOAuthFlow: vi.fn(), status: "idle", @@ -349,6 +352,28 @@ describe("CreateMCPServer", () => { expect(payload.credentials).toEqual({ auth_value: "my-secret-key" }); }); + it("does not write the browser-authorized token into form.credentials for true_passthrough", async () => { + await selectHttpTransport(); + + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "PT_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + // Simulate the browser Authorize & Fetch flow handing back an upstream token. + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "upstream-tok", token_type: "Bearer" }, undefined); + }); + + // For a browser-only mode the token must never land in form.credentials, which the OAuth flow's + // getCredentials reads for preview requests and the redirect-persist cache serializes. Without + // the guard, onTokenReceived writes it here and this returns { access_token: "upstream-tok" }. + const credentials = oauthHook.getCredentials?.() ?? {}; + expect(credentials.access_token).toBeUndefined(); + }); + it("should not show auth value field when None auth type is selected", async () => { await selectHttpTransport(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 7d160407d02..00eda92ba25 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -200,23 +200,36 @@ const CreateMCPServer: React.FC = ({ onTokenReceived: (token, registeredClient) => { setOauthAccessToken(token?.access_token ?? null); - if (token?.access_token) { - const credentials = { - access_token: token.access_token, - ...(token.refresh_token && { refresh_token: token.refresh_token }), - ...(token.expires_in && { expires_in: token.expires_in }), - ...(token.scope && { scope: token.scope }), - ...(registeredClient?.clientId && { client_id: registeredClient.clientId }), - ...(registeredClient?.clientSecret && { client_secret: registeredClient.clientSecret }), - }; - - form.setFieldsValue({ credentials }); - setAuthorizedUrl(getOAuthAuthorizationTarget(form.getFieldsValue(true))); - - NotificationsManager.success( - "OAuth authorization successful! Please click 'Create MCP Server' to save the configuration.", - ); + if (!token?.access_token) { + return; } + + if (isClientForwardedTokenMode(form.getFieldValue("auth_type"))) { + // Browser-only modes: the token is held in local state (oauthAccessToken) for tool preview + // and committed to sessionStorage on submit; it must never be written into form.credentials, + // which would persist it as server-level credentials on the created server row. Mirrors the + // edit form's onTokenReceived early return. + NotificationsManager.success( + "Token held for this browser session. Tools can now be previewed and configured; nothing will be saved to LiteLLM.", + ); + return; + } + + const credentials = { + access_token: token.access_token, + ...(token.refresh_token && { refresh_token: token.refresh_token }), + ...(token.expires_in && { expires_in: token.expires_in }), + ...(token.scope && { scope: token.scope }), + ...(registeredClient?.clientId && { client_id: registeredClient.clientId }), + ...(registeredClient?.clientSecret && { client_secret: registeredClient.clientSecret }), + }; + + form.setFieldsValue({ credentials }); + setAuthorizedUrl(getOAuthAuthorizationTarget(form.getFieldsValue(true))); + + NotificationsManager.success( + "OAuth authorization successful! Please click 'Create MCP Server' to save the configuration.", + ); }, onBeforeRedirect: persistCreateUiState, flowSource: "create", From a3f1873a8791db24e85b5db1266b0e06f8f2f6f3 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 11:04:15 -0700 Subject: [PATCH 060/101] fix(ui): extract inline object args in the MCP forms The create/edit forms passed several large object literals inline as arguments (persist-state JSON.stringify, storeMCPOAuthUserCredential, setToken, transport-clear setFieldsValue), tripping local/no-large-inline-object-arg. Assigned each to a named variable at the call site - a pure, behavior-preserving refactor verified by the create/edit suites - which lowers the whole-tree count so the eslint baseline is 512 rather than being raised to accommodate them. --- ui/litellm-dashboard/eslint-metrics.json | 2 +- .../mcp_tools/create_mcp_server.tsx | 48 +++++++++--------- .../components/mcp_tools/mcp_server_edit.tsx | 49 +++++++++---------- 3 files changed, 46 insertions(+), 53 deletions(-) diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index e4647b33d61..ded6ab97e1e 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,7 +1,7 @@ { "@typescript-eslint/no-explicit-any": 1982, "complexity": 128, - "local/no-large-inline-object-arg": 513, + "local/no-large-inline-object-arg": 512, "local/no-long-condition-chain": 233, "max-depth": 59, "no-console": 15 diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 00eda92ba25..0b39add234c 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -137,20 +137,18 @@ const CreateMCPServer: React.FC = ({ } try { const values = form.getFieldsValue(true); - setSecureItem( - CREATE_OAUTH_UI_STATE_KEY, - JSON.stringify({ - modalVisible: isModalVisible, - formValues: values, - transportType, - costConfig, - allowedTools, - hasToolAllowlistInteraction, - searchValue, - aliasManuallyEdited, - logoUrl, - }), - ); + const uiState = { + modalVisible: isModalVisible, + formValues: values, + transportType, + costConfig, + allowedTools, + hasToolAllowlistInteraction, + searchValue, + aliasManuallyEdited, + logoUrl, + }; + setSecureItem(CREATE_OAUTH_UI_STATE_KEY, JSON.stringify(uiState)); } catch (err) { console.warn("Failed to persist MCP create state", err); } @@ -510,23 +508,21 @@ const CreateMCPServer: React.FC = ({ }); if (oauthMode === "authorization_code") { const scope = oauthTokenResponse.scope; - await storeMCPOAuthUserCredential(accessToken, response.server_id, { + const oauthCredentialPayload = { access_token: oauthTokenResponse.access_token, refresh_token: oauthTokenResponse.refresh_token, expires_in: oauthTokenResponse.expires_in, scopes: typeof scope === "string" && scope ? scope.split(" ") : undefined, - }); + }; + await storeMCPOAuthUserCredential(accessToken, response.server_id, oauthCredentialPayload); } else { - setToken( - response.server_id, - { - access_token: oauthTokenResponse.access_token, - expires_in: oauthTokenResponse.expires_in, - refresh_token: oauthTokenResponse.refresh_token, - token_type: oauthTokenResponse.token_type, - }, - userID, - ); + const browserHeldToken = { + access_token: oauthTokenResponse.access_token, + expires_in: oauthTokenResponse.expires_in, + refresh_token: oauthTokenResponse.refresh_token, + token_type: oauthTokenResponse.token_type, + }; + setToken(response.server_id, browserHeldToken, userID); } } diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 2c43b9cb056..59d7b28aacb 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -180,16 +180,13 @@ const MCPServerEdit: React.FC = ({ const effectiveAuthType = form.getFieldValue("auth_type") ?? mcpServer.auth_type; if (isClientForwardedTokenMode(effectiveAuthType)) { - setToken( - mcpServer.server_id, - { - access_token: token.access_token, - expires_in: token.expires_in, - refresh_token: token.refresh_token, - token_type: token.token_type, - }, - userID, - ); + const browserHeldToken = { + access_token: token.access_token, + expires_in: token.expires_in, + refresh_token: token.refresh_token, + token_type: token.token_type, + }; + setToken(mcpServer.server_id, browserHeldToken, userID); NotificationsManager.success( "Token held for this browser session. Tools can now be loaded and configured; nothing was saved to LiteLLM.", ); @@ -467,7 +464,7 @@ const MCPServerEdit: React.FC = ({ const handleTransportChange = (value: string) => { // Clear fields that are not relevant for the selected transport. if (value === "stdio") { - form.setFieldsValue({ + const clearedForStdio = { url: undefined, spec_path: undefined, auth_type: undefined, @@ -475,15 +472,17 @@ const MCPServerEdit: React.FC = ({ authorization_url: undefined, token_url: undefined, registration_url: undefined, - }); + }; + form.setFieldsValue(clearedForStdio); } else if (value === TRANSPORT.OPENAPI) { - form.setFieldsValue({ + const clearedForOpenapi = { url: undefined, command: undefined, args: undefined, env_json: undefined, stdio_config: undefined, - }); + }; + form.setFieldsValue(clearedForOpenapi); } else { form.setFieldsValue({ spec_path: undefined, @@ -761,23 +760,21 @@ const MCPServerEdit: React.FC = ({ try { if (oauthMode === "authorization_code") { const scope = oauthTokenResponse.scope; - await storeMCPOAuthUserCredential(accessToken, mcpServer.server_id, { + const oauthCredentialPayload = { access_token: oauthTokenResponse.access_token, refresh_token: oauthTokenResponse.refresh_token, expires_in: oauthTokenResponse.expires_in, scopes: typeof scope === "string" && scope ? scope.split(" ") : undefined, - }); + }; + await storeMCPOAuthUserCredential(accessToken, mcpServer.server_id, oauthCredentialPayload); } else if (oauthMode === "passthrough") { - setToken( - mcpServer.server_id, - { - access_token: oauthTokenResponse.access_token, - expires_in: oauthTokenResponse.expires_in, - refresh_token: oauthTokenResponse.refresh_token, - token_type: oauthTokenResponse.token_type, - }, - userID, - ); + const browserHeldToken = { + access_token: oauthTokenResponse.access_token, + expires_in: oauthTokenResponse.expires_in, + refresh_token: oauthTokenResponse.refresh_token, + token_type: oauthTokenResponse.token_type, + }; + setToken(mcpServer.server_id, browserHeldToken, userID); } } catch (error: unknown) { const message = error instanceof Error ? error.message : ""; From 0a40bd7ae5e8578eba0c60fe4efb60510eb032f9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:47:51 -0700 Subject: [PATCH 061/101] fix(ui): prevent reasoning block from expanding chat playground layout (#32485) The expanded reasoning block did not constrain its width or break long unbreakable tokens, so its inline-block bubble grew past its max width and pushed the whole page wider (#32481). Mirror the message body handling by capping the container width and breaking long words/code. Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat_ui/ReasoningContent.test.tsx | 32 +++++++++++++++++++ .../components/chat_ui/ReasoningContent.tsx | 14 ++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.test.tsx diff --git a/ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.test.tsx new file mode 100644 index 00000000000..35540d3ebde --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.test.tsx @@ -0,0 +1,32 @@ +import { render, screen, fireEvent } from "@testing-library/react"; +import { describe, it, expect } from "vitest"; +import ReasoningContent from "./ReasoningContent"; + +describe("ReasoningContent", () => { + it("should render nothing when reasoningContent is empty", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it("should show reasoning content expanded by default and toggle on click", () => { + render(); + + expect(screen.getByText("thinking hard")).toBeInTheDocument(); + expect(screen.getByText("Hide reasoning")).toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button")); + + expect(screen.queryByText("thinking hard")).not.toBeInTheDocument(); + expect(screen.getByText("Show reasoning")).toBeInTheDocument(); + }); + + it("should constrain width and break long words so it cannot expand the layout (regression #32481)", () => { + const longToken = "a".repeat(500); + render(); + + const contentBox = screen.getByText(longToken).closest("div.mt-2"); + expect(contentBox).not.toBeNull(); + expect(contentBox).toHaveClass("max-w-full"); + expect(contentBox).toHaveStyle({ wordBreak: "break-word", overflowWrap: "break-word" }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.tsx b/ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.tsx index e537c6de79f..30ba2d3fd95 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/ReasoningContent.tsx @@ -27,7 +27,10 @@ const ReasoningContent: React.FC = ({ reasoningContent }) {isExpanded && ( -
+
= ({ reasoningContent }) language={match[1]} PreTag="div" className="rounded-md my-2" + wrapLines={true} + wrapLongLines={true} {...props} > {String(children).replace(/\n$/, "")} ) : ( - + {children} ); }, + pre: ({ node, ...props }) => 
,
             }}
           >
             {reasoningContent}

From 4e63c0c9e66618d5a6e2385e966a05d4da5a51ec Mon Sep 17 00:00:00 2001
From: T K Chandra Hasan 
Date: Fri, 10 Jul 2026 00:18:11 +0530
Subject: [PATCH 062/101] Fix enterprise doc link (#31815)

Signed-off-by: T K Chandra Hasan 
---
 enterprise/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/enterprise/README.md b/enterprise/README.md
index f5eb5078e81..c708dad5a06 100644
--- a/enterprise/README.md
+++ b/enterprise/README.md
@@ -6,4 +6,4 @@ Code in this folder is licensed under a commercial license. Please review the [L
 
 👉 **Using in an Enterprise / Need specific features ?** Meet with us [here](https://enterprise.litellm.ai/demo?month=2024-02)
 
-See all Enterprise Features here 👉 [Docs](https://docs.litellm.ai/docs/proxy/enterprise)
+See all Enterprise Features here 👉 [Docs](https://docs.litellm.ai/docs/enterprise)

From a874de6ac60a4c4cc940576adaf181bc4ae8494a Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Thu, 9 Jul 2026 11:51:12 -0700
Subject: [PATCH 063/101] feat(models): add GPT-5.6 (sol/terra/luna) pricing
 and metadata (#32659)

* feat(models): add GPT-5.6 (sol/terra/luna) pricing and metadata

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: allow gpt-5.6 service-tier cache-write keys in model prices schema

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix: floating point entry errors

---------

Co-authored-by: mateo 
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
---
 ...odel_prices_and_context_window_backup.json | 212 ++++++++++++++++++
 model_prices_and_context_window.json          | 212 ++++++++++++++++++
 .../llm_cost_calc/test_llm_cost_calc_utils.py |  55 +++++
 .../llms/openai/test_is_model_gpt_5_model.py  |  43 ++++
 .../test_gpt_5_6_model_metadata.py            |  79 +++++++
 tests/test_litellm/test_utils.py              |   3 +
 6 files changed, 604 insertions(+)
 create mode 100644 tests/test_litellm/test_gpt_5_6_model_metadata.py

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index db534b52df9..a111f301d11 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -22273,6 +22273,218 @@
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": true
     },
+    "gpt-5.6": {
+        "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
+        "cache_creation_input_token_cost_flex": 3.125e-06,
+        "cache_creation_input_token_cost_priority": 1.25e-05,
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
+        "cache_read_input_token_cost_flex": 2.5e-07,
+        "cache_read_input_token_cost_priority": 1e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "input_cost_per_token_batches": 2.5e-06,
+        "input_cost_per_token_flex": 2.5e-06,
+        "input_cost_per_token_priority": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "output_cost_per_token_batches": 1.5e-05,
+        "output_cost_per_token_flex": 1.5e-05,
+        "output_cost_per_token_priority": 6e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-5.6-sol": {
+        "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
+        "cache_creation_input_token_cost_flex": 3.125e-06,
+        "cache_creation_input_token_cost_priority": 1.25e-05,
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
+        "cache_read_input_token_cost_flex": 2.5e-07,
+        "cache_read_input_token_cost_priority": 1e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "input_cost_per_token_batches": 2.5e-06,
+        "input_cost_per_token_flex": 2.5e-06,
+        "input_cost_per_token_priority": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "output_cost_per_token_batches": 1.5e-05,
+        "output_cost_per_token_flex": 1.5e-05,
+        "output_cost_per_token_priority": 6e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-5.6-terra": {
+        "cache_creation_input_token_cost": 3.125e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 6.25e-06,
+        "cache_creation_input_token_cost_flex": 1.5625e-06,
+        "cache_creation_input_token_cost_priority": 6.25e-06,
+        "cache_read_input_token_cost": 2.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost_flex": 1.25e-07,
+        "cache_read_input_token_cost_priority": 5e-07,
+        "input_cost_per_token": 2.5e-06,
+        "input_cost_per_token_above_272k_tokens": 5e-06,
+        "input_cost_per_token_batches": 1.25e-06,
+        "input_cost_per_token_flex": 1.25e-06,
+        "input_cost_per_token_priority": 5e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "output_cost_per_token_above_272k_tokens": 2.25e-05,
+        "output_cost_per_token_batches": 7.5e-06,
+        "output_cost_per_token_flex": 7.5e-06,
+        "output_cost_per_token_priority": 3e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-5.6-luna": {
+        "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06,
+        "cache_creation_input_token_cost_flex": 6.25e-07,
+        "cache_creation_input_token_cost_priority": 2.5e-06,
+        "cache_read_input_token_cost": 1e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 2e-07,
+        "cache_read_input_token_cost_flex": 5e-08,
+        "cache_read_input_token_cost_priority": 2e-07,
+        "input_cost_per_token": 1e-06,
+        "input_cost_per_token_above_272k_tokens": 2e-06,
+        "input_cost_per_token_batches": 5e-07,
+        "input_cost_per_token_flex": 5e-07,
+        "input_cost_per_token_priority": 2e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_272k_tokens": 9e-06,
+        "output_cost_per_token_batches": 3e-06,
+        "output_cost_per_token_flex": 3e-06,
+        "output_cost_per_token_priority": 1.2e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
     "gpt-5.5": {
         "cache_read_input_token_cost": 5e-07,
         "cache_read_input_token_cost_above_272k_tokens": 1e-06,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 363ba9842b0..d6e4a265da0 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -22431,6 +22431,218 @@
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": true
     },
+    "gpt-5.6": {
+        "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
+        "cache_creation_input_token_cost_flex": 3.125e-06,
+        "cache_creation_input_token_cost_priority": 1.25e-05,
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
+        "cache_read_input_token_cost_flex": 2.5e-07,
+        "cache_read_input_token_cost_priority": 1e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "input_cost_per_token_batches": 2.5e-06,
+        "input_cost_per_token_flex": 2.5e-06,
+        "input_cost_per_token_priority": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "output_cost_per_token_batches": 1.5e-05,
+        "output_cost_per_token_flex": 1.5e-05,
+        "output_cost_per_token_priority": 6e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-5.6-sol": {
+        "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
+        "cache_creation_input_token_cost_flex": 3.125e-06,
+        "cache_creation_input_token_cost_priority": 1.25e-05,
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
+        "cache_read_input_token_cost_flex": 2.5e-07,
+        "cache_read_input_token_cost_priority": 1e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "input_cost_per_token_batches": 2.5e-06,
+        "input_cost_per_token_flex": 2.5e-06,
+        "input_cost_per_token_priority": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "output_cost_per_token_batches": 1.5e-05,
+        "output_cost_per_token_flex": 1.5e-05,
+        "output_cost_per_token_priority": 6e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-5.6-terra": {
+        "cache_creation_input_token_cost": 3.125e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 6.25e-06,
+        "cache_creation_input_token_cost_flex": 1.5625e-06,
+        "cache_creation_input_token_cost_priority": 6.25e-06,
+        "cache_read_input_token_cost": 2.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost_flex": 1.25e-07,
+        "cache_read_input_token_cost_priority": 5e-07,
+        "input_cost_per_token": 2.5e-06,
+        "input_cost_per_token_above_272k_tokens": 5e-06,
+        "input_cost_per_token_batches": 1.25e-06,
+        "input_cost_per_token_flex": 1.25e-06,
+        "input_cost_per_token_priority": 5e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "output_cost_per_token_above_272k_tokens": 2.25e-05,
+        "output_cost_per_token_batches": 7.5e-06,
+        "output_cost_per_token_flex": 7.5e-06,
+        "output_cost_per_token_priority": 3e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-5.6-luna": {
+        "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06,
+        "cache_creation_input_token_cost_flex": 6.25e-07,
+        "cache_creation_input_token_cost_priority": 2.5e-06,
+        "cache_read_input_token_cost": 1e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 2e-07,
+        "cache_read_input_token_cost_flex": 5e-08,
+        "cache_read_input_token_cost_priority": 2e-07,
+        "input_cost_per_token": 1e-06,
+        "input_cost_per_token_above_272k_tokens": 2e-06,
+        "input_cost_per_token_batches": 5e-07,
+        "input_cost_per_token_flex": 5e-07,
+        "input_cost_per_token_priority": 2e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_272k_tokens": 9e-06,
+        "output_cost_per_token_batches": 3e-06,
+        "output_cost_per_token_flex": 3e-06,
+        "output_cost_per_token_priority": 1.2e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
     "gpt-5.5": {
         "cache_read_input_token_cost": 5e-07,
         "cache_read_input_token_cost_above_272k_tokens": 1e-06,
diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
index 0e6c6061b46..dcc359ceff4 100644
--- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
+++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
@@ -507,6 +507,61 @@ def test_generic_cost_per_token_gpt55_pro():
     )
 
 
+@pytest.mark.parametrize(
+    "model,input_cost,output_cost,cache_read_cost,cache_write_cost",
+    [
+        ("gpt-5.6", 5e-6, 3e-5, 5e-7, 6.25e-6),
+        ("gpt-5.6-sol", 5e-6, 3e-5, 5e-7, 6.25e-6),
+        ("gpt-5.6-terra", 2.5e-6, 1.5e-5, 2.5e-7, 3.125e-6),
+        ("gpt-5.6-luna", 1e-6, 6e-6, 1e-7, 1.25e-6),
+    ],
+)
+def test_generic_cost_per_token_gpt56(
+    model, input_cost, output_cost, cache_read_cost, cache_write_cost
+):
+    """gpt-5.6 (sol/terra/luna): base pricing + new cache-write cost.
+
+    Cache writes are billed at 1.25x the uncached input rate for this family.
+    """
+    custom_llm_provider = "openai"
+    os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+
+    model_cost_map = litellm.model_cost[model]
+
+    assert model_cost_map["input_cost_per_token"] == input_cost
+    assert model_cost_map["output_cost_per_token"] == output_cost
+    assert model_cost_map["cache_read_input_token_cost"] == cache_read_cost
+    assert model_cost_map["cache_creation_input_token_cost"] == cache_write_cost
+    assert model_cost_map["litellm_provider"] == "openai"
+    assert model_cost_map["mode"] == "chat"
+    assert model_cost_map["cache_creation_input_token_cost"] == pytest.approx(
+        input_cost * 1.25
+    )
+    assert model_cost_map["max_input_tokens"] == 1050000
+    assert model_cost_map["input_cost_per_token_above_272k_tokens"] == pytest.approx(
+        input_cost * 2
+    )
+    assert model_cost_map["output_cost_per_token_above_272k_tokens"] == pytest.approx(
+        output_cost * 1.5
+    )
+
+    prompt_tokens = 1000
+    completion_tokens = 500
+    usage = Usage(
+        prompt_tokens=prompt_tokens,
+        completion_tokens=completion_tokens,
+        total_tokens=prompt_tokens + completion_tokens,
+    )
+    prompt_cost, completion_cost = generic_cost_per_token(
+        model=model,
+        usage=usage,
+        custom_llm_provider=custom_llm_provider,
+    )
+    assert round(prompt_cost, 10) == round(input_cost * prompt_tokens, 10)
+    assert round(completion_cost, 10) == round(output_cost * completion_tokens, 10)
+
+
 @pytest.mark.parametrize(
     "model,expected_none,expected_xhigh,expected_minimal",
     [
diff --git a/tests/test_litellm/llms/openai/test_is_model_gpt_5_model.py b/tests/test_litellm/llms/openai/test_is_model_gpt_5_model.py
index 02dd9dade0a..1095819c98c 100644
--- a/tests/test_litellm/llms/openai/test_is_model_gpt_5_model.py
+++ b/tests/test_litellm/llms/openai/test_is_model_gpt_5_model.py
@@ -50,6 +50,10 @@ GPT5_MODELS = [
     "gpt-5.5-pro",
     "gpt-5.5-2026-04-23",  # dated variant
     "gpt-5.5-pro-2026-04-23",  # dated variant
+    "gpt-5.6",
+    "gpt-5.6-sol",
+    "gpt-5.6-terra",
+    "gpt-5.6-luna",
     "gpt-5.1-chat",  # versioned chat — THE KEY REGRESSION CASE
     "gpt-5.2-chat",  # versioned chat — also a regression case
     "gpt-5.3-chat",  # versioned chat — THE KEY REGRESSION CASE
@@ -112,6 +116,45 @@ class TestOpenAIGPT5ConfigIsModelGpt5Model:
             ), f"Expected '{model}' (gpt-5-chat family) NOT to be on the GPT-5 path"
 
 
+# Models that are gpt-5.4 or newer. main.py gates the automatic switch to the
+# /v1/responses bridge (when reasoning_effort is set and tools are passed) on
+# is_model_gpt_5_4_plus_model, so the gpt-5.6 family must land on the True side.
+GPT5_4_PLUS_MODELS = [
+    "gpt-5.4",
+    "gpt-5.5",
+    "gpt-5.5-pro",
+    "gpt-5.6",
+    "gpt-5.6-sol",
+    "gpt-5.6-terra",
+    "gpt-5.6-luna",
+    "openai/gpt-5.6-sol",
+]
+
+GPT5_PRE_5_4_MODELS = [
+    "gpt-5",
+    "gpt-5.1",
+    "gpt-5.2",
+    "gpt-5.3",
+    "gpt-5.3-chat",
+    "gpt-4o",
+]
+
+
+class TestOpenAIGPT5ConfigIsModelGpt54PlusModel:
+
+    @pytest.mark.parametrize("model", GPT5_4_PLUS_MODELS)
+    def test_gpt5_4_plus_models_are_classified_as_5_4_plus(self, model: str):
+        assert OpenAIGPT5Config.is_model_gpt_5_4_plus_model(
+            model
+        ), f"Expected '{model}' to be classified as gpt-5.4-or-newer"
+
+    @pytest.mark.parametrize("model", GPT5_PRE_5_4_MODELS)
+    def test_pre_5_4_models_are_not_classified_as_5_4_plus(self, model: str):
+        assert not OpenAIGPT5Config.is_model_gpt_5_4_plus_model(
+            model
+        ), f"Expected '{model}' NOT to be classified as gpt-5.4-or-newer"
+
+
 # ---------------------------------------------------------------------------
 # AzureOpenAIGPT5Config
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/test_gpt_5_6_model_metadata.py b/tests/test_litellm/test_gpt_5_6_model_metadata.py
new file mode 100644
index 00000000000..30a0777f477
--- /dev/null
+++ b/tests/test_litellm/test_gpt_5_6_model_metadata.py
@@ -0,0 +1,79 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+
+GPT_5_6_MODELS = ("gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna")
+
+STANDARD_PRICING = {
+    "gpt-5.6": (5e-06, 3e-05, 5e-07, 6.25e-06),
+    "gpt-5.6-sol": (5e-06, 3e-05, 5e-07, 6.25e-06),
+    "gpt-5.6-terra": (2.5e-06, 1.5e-05, 2.5e-07, 3.125e-06),
+    "gpt-5.6-luna": (1e-06, 6e-06, 1e-07, 1.25e-06),
+}
+
+
+@pytest.mark.parametrize("model", GPT_5_6_MODELS)
+def test_openai_gpt_5_6_model_info(model):
+    json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
+    with open(json_path) as f:
+        model_cost = json.load(f)
+
+    info = model_cost.get(model)
+    assert info is not None, f"{model} not found in model_prices_and_context_window.json"
+
+    assert info["litellm_provider"] == "openai"
+    assert info["mode"] == "chat"
+
+    input_cost, output_cost, cache_read_cost, cache_write_cost = STANDARD_PRICING[model]
+    assert info["input_cost_per_token"] == input_cost
+    assert info["output_cost_per_token"] == output_cost
+    assert info["cache_read_input_token_cost"] == cache_read_cost
+    assert info["cache_creation_input_token_cost"] == cache_write_cost
+    assert info["cache_creation_input_token_cost"] == pytest.approx(input_cost * 1.25)
+
+    assert info["input_cost_per_token_above_272k_tokens"] == pytest.approx(input_cost * 2)
+    assert info["output_cost_per_token_above_272k_tokens"] == pytest.approx(output_cost * 1.5)
+    assert info["cache_read_input_token_cost_above_272k_tokens"] == pytest.approx(cache_read_cost * 2)
+
+    assert info["max_input_tokens"] == 1050000
+    assert info["max_output_tokens"] == 128000
+    assert info["max_tokens"] == 128000
+
+    assert info["supports_function_calling"] is True
+    assert info["supports_prompt_caching"] is True
+    assert info["supports_reasoning"] is True
+    assert info["supports_response_schema"] is True
+    assert info["supports_tool_choice"] is True
+    assert info["supports_vision"] is True
+    assert info["supports_web_search"] is True
+    assert info["supports_none_reasoning_effort"] is True
+    assert info["supports_xhigh_reasoning_effort"] is True
+    assert info["supports_minimal_reasoning_effort"] is False
+
+    assert info["supported_endpoints"] == ["/v1/chat/completions", "/v1/batch", "/v1/responses"]
+    assert info["supported_modalities"] == ["text", "image"]
+    assert info["supported_output_modalities"] == ["text"]
+
+    routed_model, provider, _, _ = get_llm_provider(model=f"openai/{model}")
+    assert routed_model == model
+    assert provider == "openai"
+
+
+def test_gpt_5_6_backup_matches_main():
+    """Ensure the bundled model cost map stays in sync with the canonical file."""
+    repo_root = Path(__file__).parents[2]
+    main_path = repo_root / "model_prices_and_context_window.json"
+    backup_path = repo_root / "litellm" / "model_prices_and_context_window_backup.json"
+
+    with open(main_path) as f:
+        main_cost = json.load(f)
+    with open(backup_path) as f:
+        backup_cost = json.load(f)
+
+    for model in GPT_5_6_MODELS:
+        assert backup_cost.get(model) == main_cost.get(model), (
+            f"{model} differs between main and backup model cost maps"
+        )
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 053fe970d3e..47be8cb2eec 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -710,6 +710,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                 "cache_creation_input_token_cost": {"type": "number"},
                 "cache_creation_input_token_cost_above_1hr": {"type": "number"},
                 "cache_creation_input_token_cost_above_200k_tokens": {"type": "number"},
+                "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"},
+                "cache_creation_input_token_cost_flex": {"type": "number"},
+                "cache_creation_input_token_cost_priority": {"type": "number"},
                 "cache_read_input_token_cost": {"type": "number"},
                 "cache_read_input_token_cost_above_200k_tokens": {"type": "number"},
                 "cache_read_input_token_cost_above_272k_tokens": {"type": "number"},

From 7d63b86e00cf0b4acabf5f582eef6a9e904c57e6 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri 
Date: Thu, 9 Jul 2026 11:59:16 -0700
Subject: [PATCH 064/101] fix(ui): forward refs through ui primitives and fail
 tests on swallowed refs (#32401)

* fix(ui): forward refs through ui primitives and fail tests on swallowed refs

Under React 18 a ref passed to a plain function component is dropped
with only a dev console warning, so Base UI render-prop triggers
composed over our shadcn-style primitives silently stop working (the
tooltip just never opens; ui/badge.tsx hit exactly this on the shared
DataTable branch). Label, Separator, Skeleton, UiLoadingSpinner and the
Table family now use React.forwardRef like Button and Input already
did, a contract test pins ref delivery for each, and setupTests turns
React's ref warning into a test failure so the next primitive that
swallows a ref fails CI instead of shipping a dead tooltip

* fix(ui): include captured ref warnings in the tripwire error

The afterEach tripwire threw a fixed message and discarded the collected
React warnings, so a failure never said which component swallowed the ref.
Append the captured warnings (component name + stack) to the thrown error.
---
 .../src/components/ui/label.tsx               |  10 +-
 .../src/components/ui/ref-forwarding.test.tsx | 103 ++++++++++++++++++
 .../src/components/ui/separator.tsx           |  11 +-
 .../src/components/ui/skeleton.tsx            |  11 +-
 .../src/components/ui/table.tsx               |  85 +++++++++------
 .../src/components/ui/ui-loading-spinner.tsx  |  68 ++++++------
 ui/litellm-dashboard/tests/setupTests.ts      |  22 ++++
 7 files changed, 235 insertions(+), 75 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx

diff --git a/ui/litellm-dashboard/src/components/ui/label.tsx b/ui/litellm-dashboard/src/components/ui/label.tsx
index ded2dfc1a7b..1ac4eed0d4e 100644
--- a/ui/litellm-dashboard/src/components/ui/label.tsx
+++ b/ui/litellm-dashboard/src/components/ui/label.tsx
@@ -4,9 +4,10 @@ import * as React from "react";
 
 import { cn } from "@/lib/cva.config";
 
-function Label({ className, ...props }: React.ComponentProps<"label">) {
-  return (
+const Label = React.forwardRef>(
+  ({ className, ...props }, ref) => (
+ caption + + + h + + + + + d + + + + + f + + +
, + ); + + expect(table.current).toBeInstanceOf(HTMLTableElement); + expect(caption.current).toBeInstanceOf(HTMLTableCaptionElement); + expect(header.current?.tagName).toBe("THEAD"); + expect(body.current?.tagName).toBe("TBODY"); + expect(footer.current?.tagName).toBe("TFOOT"); + expect(row.current).toBeInstanceOf(HTMLTableRowElement); + expect(head.current?.tagName).toBe("TH"); + expect(cell.current?.tagName).toBe("TD"); + }); +}); + +describe("setupTests ref tripwire", () => { + it("records a violation when a ref is passed to a plain function component", () => { + const Plain = (props: React.ComponentPropsWithoutRef<"span">) => ; + const ref = React.createRef(); + render(React.createElement(Plain as never, { ref })); + const consume = (globalThis as { __consumePendingRefWarnings?: () => string[] }).__consumePendingRefWarnings; + expect(consume).toBeDefined(); + const violations = consume!(); + expect(violations).toHaveLength(1); + expect(violations[0]).toContain("Function components cannot be given refs"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ui/separator.tsx b/ui/litellm-dashboard/src/components/ui/separator.tsx index 443f8e905f9..a8a8d9cf5c2 100644 --- a/ui/litellm-dashboard/src/components/ui/separator.tsx +++ b/ui/litellm-dashboard/src/components/ui/separator.tsx @@ -1,12 +1,14 @@ "use client"; import { Separator as SeparatorPrimitive } from "@base-ui/react/separator"; +import * as React from "react"; import { cn } from "@/lib/cva.config"; -function Separator({ className, orientation = "horizontal", ...props }: SeparatorPrimitive.Props) { - return ( +const Separator = React.forwardRef, SeparatorPrimitive.Props>( + ({ className, orientation = "horizontal", ...props }, ref) => ( - ); -} + ), +); +Separator.displayName = "Separator"; export { Separator }; diff --git a/ui/litellm-dashboard/src/components/ui/skeleton.tsx b/ui/litellm-dashboard/src/components/ui/skeleton.tsx index e27145708a2..69ff4891cec 100644 --- a/ui/litellm-dashboard/src/components/ui/skeleton.tsx +++ b/ui/litellm-dashboard/src/components/ui/skeleton.tsx @@ -1,7 +1,12 @@ +import * as React from "react"; + import { cn } from "@/lib/cva.config"; -function Skeleton({ className, ...props }: React.ComponentProps<"div">) { - return
; -} +const Skeleton = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +Skeleton.displayName = "Skeleton"; export { Skeleton }; diff --git a/ui/litellm-dashboard/src/components/ui/table.tsx b/ui/litellm-dashboard/src/components/ui/table.tsx index aff687f432d..6271a9e89ac 100644 --- a/ui/litellm-dashboard/src/components/ui/table.tsx +++ b/ui/litellm-dashboard/src/components/ui/table.tsx @@ -4,35 +4,45 @@ import * as React from "react"; import { cn } from "@/lib/cva.config"; -function Table({ className, ...props }: React.ComponentProps<"table">) { - return ( +const Table = React.forwardRef>( + ({ className, ...props }, ref) => (
- +
- ); -} + ), +); +Table.displayName = "Table"; -function TableHeader({ className, ...props }: React.ComponentProps<"thead">) { - return ; -} +const TableHeader = React.forwardRef>( + ({ className, ...props }, ref) => ( + + ), +); +TableHeader.displayName = "TableHeader"; -function TableBody({ className, ...props }: React.ComponentProps<"tbody">) { - return ; -} +const TableBody = React.forwardRef>( + ({ className, ...props }, ref) => ( + + ), +); +TableBody.displayName = "TableBody"; -function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) { - return ( +const TableFooter = React.forwardRef>( + ({ className, ...props }, ref) => ( tr]:last:border-b-0", className)} {...props} /> - ); -} + ), +); +TableFooter.displayName = "TableFooter"; -function TableRow({ className, ...props }: React.ComponentProps<"tr">) { - return ( +const TableRow = React.forwardRef>( + ({ className, ...props }, ref) => ( ) { )} {...props} /> - ); -} + ), +); +TableRow.displayName = "TableRow"; -function TableHead({ className, ...props }: React.ComponentProps<"th">) { - return ( +const TableHead = React.forwardRef>( + ({ className, ...props }, ref) => (
[role=checkbox]]:translate-y-[2px]", @@ -53,12 +65,14 @@ function TableHead({ className, ...props }: React.ComponentProps<"th">) { )} {...props} /> - ); -} + ), +); +TableHead.displayName = "TableHead"; -function TableCell({ className, ...props }: React.ComponentProps<"td">) { - return ( +const TableCell = React.forwardRef>( + ({ className, ...props }, ref) => ( [role=checkbox]]:translate-y-[2px]", @@ -66,13 +80,20 @@ function TableCell({ className, ...props }: React.ComponentProps<"td">) { )} {...props} /> - ); -} + ), +); +TableCell.displayName = "TableCell"; -function TableCaption({ className, ...props }: React.ComponentProps<"caption">) { - return ( -
- ); -} +const TableCaption = React.forwardRef>( + ({ className, ...props }, ref) => ( + + ), +); +TableCaption.displayName = "TableCaption"; export { Table, TableHeader, TableBody, TableFooter, TableHead, TableRow, TableCell, TableCaption }; diff --git a/ui/litellm-dashboard/src/components/ui/ui-loading-spinner.tsx b/ui/litellm-dashboard/src/components/ui/ui-loading-spinner.tsx index 09e52ef0d48..5fd62d92973 100644 --- a/ui/litellm-dashboard/src/components/ui/ui-loading-spinner.tsx +++ b/ui/litellm-dashboard/src/components/ui/ui-loading-spinner.tsx @@ -4,39 +4,43 @@ import { cx } from "@/lib/cva.config"; type LoadingSpinnerProps = React.SVGProps; -export function UiLoadingSpinner({ className = "", ...props }: LoadingSpinnerProps) { - const id = useId(); +export const UiLoadingSpinner = React.forwardRef( + ({ className = "", ...props }, ref) => { + const id = useId(); - useSafeLayoutEffect(() => { - const animations = document - .getAnimations() - .filter((a) => a instanceof CSSAnimation && a.animationName === "spin") as CSSAnimation[]; + useSafeLayoutEffect(() => { + const animations = document + .getAnimations() + .filter((a) => a instanceof CSSAnimation && a.animationName === "spin") as CSSAnimation[]; - const self = animations.find((a) => (a.effect as KeyframeEffect).target?.getAttribute("data-spinner-id") === id); + const self = animations.find((a) => (a.effect as KeyframeEffect).target?.getAttribute("data-spinner-id") === id); - const anyOther = animations.find( - (a) => a.effect instanceof KeyframeEffect && a.effect.target?.getAttribute("data-spinner-id") !== id, + const anyOther = animations.find( + (a) => a.effect instanceof KeyframeEffect && a.effect.target?.getAttribute("data-spinner-id") !== id, + ); + + if (self && anyOther) { + self.currentTime = anyOther.currentTime; + } + }, [id]); + + return ( + + + + ); - - if (self && anyOther) { - self.currentTime = anyOther.currentTime; - } - }, [id]); - - return ( - - - - - ); -} + }, +); +UiLoadingSpinner.displayName = "UiLoadingSpinner"; diff --git a/ui/litellm-dashboard/tests/setupTests.ts b/ui/litellm-dashboard/tests/setupTests.ts index 6bc2e7775a1..69506a9f246 100644 --- a/ui/litellm-dashboard/tests/setupTests.ts +++ b/ui/litellm-dashboard/tests/setupTests.ts @@ -149,8 +149,30 @@ vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ }), })); +const pendingRefWarnings: string[] = []; +const consumePendingRefWarnings = (): string[] => pendingRefWarnings.splice(0, pendingRefWarnings.length); +(globalThis as { __consumePendingRefWarnings?: () => string[] }).__consumePendingRefWarnings = + consumePendingRefWarnings; + +const originalConsoleError = console.error.bind(console); +vi.spyOn(console, "error").mockImplementation((...args: unknown[]) => { + originalConsoleError(...args); + if (typeof args[0] === "string" && args[0].includes("Function components cannot be given refs")) { + pendingRefWarnings.push(args.map(String).join(" ")); + } +}); + afterEach(() => { cleanup(); + const refWarnings = consumePendingRefWarnings(); + if (refWarnings.length > 0) { + throw new Error( + "A ref was passed to a plain function component and silently dropped under React 18, which breaks " + + "ref-based composition (Base UI render triggers, tooltips, focus). Wrap the component in React.forwardRef. " + + "This tripwire lives in tests/setupTests.ts and can be removed after the React 19 upgrade.\n\n" + + refWarnings.join("\n\n"), + ); + } }); // Make toLocaleString deterministic in tests; individual tests can override From 1d9a86eac40fca902673ba57c613d6d9a6febe37 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 9 Jul 2026 11:59:22 -0700 Subject: [PATCH 065/101] refactor(ui): consolidate invitation flow into the dashboard layout (#32576) The App Router migration is complete: every page is a path route and the legacy `?page=` switch is gone from the index. This closes it out. The `/ui/` index (page.tsx) kept its own duplicate copy of teams state, a teams fetch, and keys/addKey plumbing solely to feed a second `UserDashboard` render for the `invitation_id` case. That was redundant: `ApiKeysDashboard` already renders `UserDashboard` sourcing its own data, so the index is thinned to just render ``. The login redirect, the legacy `?page=` deep-link redirect for old bookmarks, and the post-login return-URL handling stay on the index. The invitation entry point now resolves in one place. Modern invitation links already point at the dedicated `/onboarding` route; the dashboard layout now redirects legacy `/ui/?invitation_id=` links there too (via `migratedHref`, the same base-aware redirect the index uses for `?page=`), instead of re-rendering that route's page component inline. This removes an import of one route's `page.tsx` into another module, and lets the now-unreachable `if (invitation_id) return ` branch in the shared `user_dashboard.tsx` be deleted along with its dead `Onboarding` import and `searchParams` read. A layout test asserts the redirect and fails if it regresses. `legacyPageHref` and the sidebar's migrated-vs-legacy href fallback are left in place; they are still live for the parent-category nav nodes (agentic, tools, experimental, settings) that are not page routes. eslint-metrics.json is resynced: -2 no-explicit-any from the removed `any` casts, plus pre-existing drift the gate requires the snapshot to match. --- ui/litellm-dashboard/eslint-metrics.json | 2 +- .../src/app/(dashboard)/layout.test.tsx | 30 ++++++++- .../src/app/(dashboard)/layout.tsx | 13 +++- .../src/app/(dashboard)/page.tsx | 62 +++---------------- .../src/components/user_dashboard.tsx | 11 ---- 5 files changed, 47 insertions(+), 71 deletions(-) diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index 37bad071081..fcf60934f64 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,5 +1,5 @@ { - "@typescript-eslint/no-explicit-any": 1982, + "@typescript-eslint/no-explicit-any": 1980, "complexity": 128, "local/no-large-inline-object-arg": 519, "local/no-long-condition-chain": 233, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx index 7573ddb5a0f..92a1d40b0e3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.test.tsx @@ -3,9 +3,13 @@ import { render, screen, waitFor } from "@testing-library/react"; import { AuthProvider } from "@/contexts/AuthContext"; import Layout from "./layout"; +const { replaceMock } = vi.hoisted(() => ({ replaceMock: vi.fn() })); + +let searchParamsValue = new URLSearchParams(); + vi.mock("next/navigation", () => ({ - useRouter: vi.fn(() => ({ push: vi.fn(), replace: vi.fn() })), - useSearchParams: vi.fn(() => new URLSearchParams()), + useRouter: vi.fn(() => ({ push: vi.fn(), replace: replaceMock })), + useSearchParams: vi.fn(() => searchParamsValue), usePathname: vi.fn(() => "/ui/guardrails"), })); @@ -58,6 +62,7 @@ describe("(dashboard) Layout", () => { beforeEach(() => { vi.clearAllMocks(); pendingUiConfig = createDeferred(); + searchParamsValue = new URLSearchParams(); }); it("does not mount route content until getUiConfig has resolved", async () => { @@ -79,4 +84,25 @@ describe("(dashboard) Layout", () => { expect(screen.getByTestId("navbar")).toBeTruthy(); expect(screen.queryByTestId("loading-screen")).toBeNull(); }); + + it("redirects an invitation link to the onboarding route instead of rendering the dashboard shell", async () => { + searchParamsValue = new URLSearchParams("invitation_id=abc123"); + + render( + + +
+ + , + ); + + pendingUiConfig.resolve(); + + await waitFor(() => + expect(replaceMock).toHaveBeenCalledWith(expect.stringContaining("/onboarding?invitation_id=abc123")), + ); + expect(screen.queryByTestId("page-content")).toBeNull(); + expect(screen.queryByTestId("navbar")).toBeNull(); + expect(screen.queryByTestId("sidebar")).toBeNull(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index c84209b80cf..b8eb4e66ed0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -137,17 +137,26 @@ function DashboardShell({ children }: { children: React.ReactNode }) { } function LayoutContent({ children }: { children: React.ReactNode }) { + const router = useRouter(); const searchParams = useSearchParams(); const { accessToken, authLoading } = useAuth(); const isInvitationFlow = Boolean(searchParams.get("invitation_id")); - if (authLoading) { + // Legacy invitation links point at /ui/?invitation_id=; the onboarding form now lives at its own + // /onboarding route. Redirect once ui-config has loaded so migratedHref resolves the SERVER_ROOT_PATH base. + useEffect(() => { + if (!authLoading && isInvitationFlow) { + router.replace(`${migratedHref("onboarding")}?${searchParams.toString()}`); + } + }, [authLoading, isInvitationFlow, router, searchParams]); + + if (authLoading || isInvitationFlow) { return ; } return ( - {isInvitationFlow ? children : {children}} + {children} ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx index cb4a4a0de03..6c0d780183a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/page.tsx @@ -1,11 +1,8 @@ "use client"; import ApiKeysDashboard from "@/app/(dashboard)/api-keys/ApiKeysDashboard"; -import { teamListCall as v2TeamListCall } from "@/app/(dashboard)/hooks/teams/useTeams"; import LoadingScreen from "@/components/common_components/LoadingScreen"; -import { Team } from "@/components/key_team_helpers/key_list"; import { proxyBaseUrl } from "@/components/networking"; -import UserDashboard from "@/components/user_dashboard"; import { useAuth } from "@/contexts/AuthContext"; import { buildLoginUrlWithReturn, @@ -16,32 +13,20 @@ import { } from "@/utils/returnUrlUtils"; import { MIGRATED_PAGES, migratedHref } from "@/utils/migratedPages"; import { useRouter, useSearchParams } from "next/navigation"; -import { Suspense, useEffect, useRef, useState } from "react"; +import { Suspense, useEffect, useRef } from "react"; function CreateKeyPageContent() { - const { authLoading, token, userID, userRole, userEmail, accessToken, premiumUser, setUserRole, setUserEmail } = - useAuth(); - - const [teams, setTeams] = useState(null); - const [keys, setKeys] = useState([]); + const { authLoading, token } = useAuth(); const router = useRouter(); const searchParams = useSearchParams()!; - const [createClicked, setCreateClicked] = useState(false); - - const invitation_id = searchParams.get("invitation_id"); const explicitPage = searchParams.get("page"); - const page = explicitPage || "api-keys"; // Track if we've already attempted a return URL redirect to prevent race conditions const hasAttemptedReturnRedirectRef = useRef(false); - const addKey = (data: any) => { - setKeys((prevData) => (prevData ? [...prevData, data] : [data])); - setCreateClicked(() => !createClicked); - }; - const redirectToLogin = authLoading === false && token === null && invitation_id === null; + const redirectToLogin = authLoading === false && token === null; useEffect(() => { if (redirectToLogin) { @@ -55,15 +40,13 @@ function CreateKeyPageContent() { } }, [redirectToLogin]); - // Redirect legacy query-param pages to their new path-based routes. Only when the page is - // explicitly requested via ?page=, so the bare landing renders inline and the post-login - // return-URL handling below stays intact. + // Redirect legacy ?page= deep links (old bookmarks) to their path-based routes. const isLegacyRedirect = explicitPage !== null && explicitPage in MIGRATED_PAGES; useEffect(() => { if (!authLoading && isLegacyRedirect) { - router.replace(migratedHref(MIGRATED_PAGES[page])); + router.replace(migratedHref(MIGRATED_PAGES[explicitPage])); } - }, [authLoading, isLegacyRedirect, page, router]); + }, [authLoading, isLegacyRedirect, explicitPage, router]); // Check for a stored return URL after successful authentication // This handles the case where user comes back from SSO and we need to redirect to the original URL @@ -102,42 +85,11 @@ function CreateKeyPageContent() { } }, [token]); - useEffect(() => { - if (accessToken && userID && userRole) { - v2TeamListCall(accessToken, 1, 100, { - userID: userRole !== "Admin" && userRole !== "Admin Viewer" ? userID : null, - }) - .then((response) => setTeams(response.teams ?? [])) - .catch(console.error); - } - }, [accessToken, userID, userRole]); - if (authLoading || redirectToLogin || isLegacyRedirect) { return ; } - return ( - <> - {invitation_id ? ( - - ) : ( - - )} - - ); + return ; } export default function CreateKeyPage() { diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index acb9333b051..689b5680fcc 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -2,9 +2,7 @@ import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; import { Col, Grid } from "@tremor/react"; import { jwtDecode } from "jwt-decode"; -import { useSearchParams } from "next/navigation"; import React, { useEffect, useState } from "react"; -import Onboarding from "../app/onboarding/page"; import { fetchTeams } from "./common_components/fetch_teams"; import { KeyResponse, Team } from "./key_team_helpers/key_list"; import { @@ -76,13 +74,8 @@ const UserDashboard: React.FC = ({ const [userSpendData, setUserSpendData] = useState(null); const [currentOrg, setCurrentOrg] = useState(null); - // Assuming useSearchParams() hook exists and works in your setup - const searchParams = useSearchParams()!; - const token = getCookie("token"); - const invitation_id = searchParams.get("invitation_id"); - const [accessToken, setAccessToken] = useState(null); const [teamSpend, setTeamSpend] = useState(null); const [userModels, setUserModels] = useState([]); @@ -232,10 +225,6 @@ const UserDashboard: React.FC = ({ } }, [selectedTeam]); - if (invitation_id != null) { - return ; - } - function gotoLogin() { // Clear token cookies using the utility function clearTokenCookies(); From 6eed38bcfb4e1c95fb250524117fa29e31da2dfe Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 9 Jul 2026 13:18:51 -0700 Subject: [PATCH 066/101] fix(guardrails/bedrock): honor disable_exception_on_block by raising ModifyResponseException (#32289) * fix(guardrails/bedrock): honor disable_exception_on_block by raising ModifyResponseException The Bedrock-specific GuardrailInterventionNormalStringError predates the unified guardrails refactor and no proxy code path handles it, so a block with the flag set surfaced as an uncaught Exception -> HTTP 500 in pre_call mode and was silently discarded in during_call mode (model call proceeded in the parallel asyncio.gather; the block hook's data["mock_response"] mutation happened after route_request had already unpacked kwargs). Convert the block to ModifyResponseException at the raise site inside make_bedrock_api_request. That exception is the industry-standard proxy contract already caught in proxy_server, anthropic_endpoints, response_api _endpoints, and pass_through_endpoints; it turns into a 200 response with finish_reason=content_filter and the block message as content, which is exactly what the flag was documented to yield. Post-call blocks attach the LLM response to original_response so the synthetic reply reports the upstream call's real token usage instead of zero. Deletes the now-orphaned GuardrailInterventionNormalStringError class and the dead create_guardrail_blocked_response / mock_response plumbing in the Bedrock hooks; updates the existing tests that had locked in the buggy contract. Resolves LIT-4186 * chore(guardrails/bedrock): drop dead str branch in _update_messages_with_updated_bedrock_guardrail_response Follow-up to the disable_exception_on_block fix. That method used to receive either a BedrockGuardrailResponse or a plain string (the block message, when the flag was set). Now that a block always raises ModifyResponseException before this method runs, the string branch is unreachable; tighten the type to BedrockGuardrailResponse and delete the guard. * fix(guardrails/bedrock): streaming post_call block yields synthetic stream instead of surfacing as SSE 500 Regression from the LIT-4186 refactor: pre-refactor, the streaming post_call iterator caught GuardrailInterventionNormalStringError locally and replaced the assembled response with a synthetic content-filter message, then re-emitted it as chunks via MockResponseIterator. After the refactor the exception was re-raised as ModifyResponseException, which async_streaming_data_generator serializes as a proxy 500 error frame because the SSE response headers are already flushed by the time the block fires. Non-streaming paths still let ModifyResponseException propagate to the endpoint handler (which converts it into a 200). Streaming can't do that, so keep the local synthesis: on the exception, rebind the assembled response to a ModelResponse whose single choice carries the block message as content and finish_reason=content_filter, and let the downstream MockResponseIterator emit it as chunks. Same shape a non-streaming block produces. Adds a mapped-file regression test that mutation-kills the raise behavior and locks in the synthetic-stream contract. * fix(guardrails/bedrock): preserve upstream usage on streaming post_call block Non-streaming post_call blocks report the upstream LLM call's real token usage via ModifyResponseException.original_response, which the endpoint handler unwraps through _blocked_response_usage. Streaming post_call synthesizes its own ModelResponse locally (the exception can't escape the SSE generator), and previously left .usage unset, so the client saw accurate billing on non-streaming blocks and zero on streaming blocks -- silent revenue leak. Copy the assembled response's .usage onto the synthetic block response before yielding. Pre-refactor code had the same gap (create_guardrail_blocked_response never set usage); this is a net improvement, not a regression fix. --- litellm/exceptions.py | 14 - .../guardrail_hooks/bedrock_guardrails.py | 145 ++++--- .../test_bedrock_apply_guardrail.py | 22 +- .../test_bedrock_guardrails.py | 66 ++- .../test_bedrock_guardrails.py | 405 ++++++++++++++++++ 5 files changed, 529 insertions(+), 123 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index adf7b3ef05a..aca3fb551cc 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1180,20 +1180,6 @@ class ModifyResponseException(Exception): super().__init__(message) -class GuardrailInterventionNormalStringError( - Exception -): # custom exception to raise when a guardrail intervenes, but we want to return a normal string to the user - def __init__(self, message: str): - self.message = message - super().__init__(self.message) - - def __str__(self): - return self.message - - def __repr__(self): - return self.__str__() - - class SensitiveDataRouteException(Exception): """ Exception raised when a guardrail detects sensitive data and wants to reroute the request. diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 6e46f971dd8..a45719d2eb8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -33,7 +33,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys from litellm.caching import DualCache -from litellm.exceptions import GuardrailInterventionNormalStringError +from litellm.exceptions import ModifyResponseException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.custom_httpx.http_handler import ( @@ -754,7 +754,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) bedrock_guardrail_response = BedrockGuardrailResponse(**_json_response) if self._should_raise_guardrail_blocked_exception(bedrock_guardrail_response): - raise self._get_http_exception_for_blocked_guardrail(bedrock_guardrail_response) + raise self._get_http_exception_for_blocked_guardrail( + bedrock_guardrail_response, request_data=request_data + ) else: status_code, detail_message = self._parse_bedrock_guardrail_error_response(httpx_response) verbose_proxy_logger.error( @@ -1027,8 +1029,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return blocked def _get_http_exception_for_blocked_guardrail( - self, response: BedrockGuardrailResponse - ) -> Union[HTTPException, GuardrailInterventionNormalStringError]: + self, response: BedrockGuardrailResponse, request_data: Optional[dict] = None + ) -> Union[HTTPException, ModifyResponseException]: """ Get the HTTP exception for a blocked guardrail. """ @@ -1040,7 +1042,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): bedrock_guardrail_output_text += output.get("text") or "" if self.disable_exception_on_block is True: - return GuardrailInterventionNormalStringError(message=bedrock_guardrail_output_text) + _request_data = request_data or {} + return ModifyResponseException( + message=bedrock_guardrail_output_text, + model=_request_data.get("model", "bedrock-guardrail"), + request_data=_request_data, + guardrail_name=self.guardrail_name, + ) detail: Dict[str, Any] = { "error": "Violated guardrail policy", @@ -1134,18 +1142,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # This means all actions were ANONYMIZED or NONE, so don't raise exception return False - def create_guardrail_blocked_response(self, response: str) -> ModelResponse: - from litellm.types.utils import Choices, Message, ModelResponse - - return ModelResponse( - choices=[ - Choices( - message=Message(content=response), - ) - ], - model="bedrock-guardrail", - ) - async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -1183,16 +1179,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ######################################################### ########## 1. Make the Bedrock API request ########## ######################################################### - bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = None - try: - bedrock_guardrail_response = await self.make_bedrock_api_request( - source="INPUT", - messages=filtered_messages, - request_data=data, - logging_event_type=GuardrailEventHooks.pre_call, - ) - except GuardrailInterventionNormalStringError as e: - bedrock_guardrail_response = e.message + # A block with disable_exception_on_block=True raises ModifyResponseException + # from make_bedrock_api_request; that propagates to the endpoint handler, + # which returns a 200 whose message is the guardrail's blockedInputMessaging. + bedrock_guardrail_response = await self.make_bedrock_api_request( + source="INPUT", + messages=filtered_messages, + request_data=data, + logging_event_type=GuardrailEventHooks.pre_call, + ) ######################################################### ######################################################### @@ -1207,8 +1202,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): updated_target_messages=updated_subset, target_indices=filter_result.target_indices, ) - if isinstance(bedrock_guardrail_response, str): - data["mock_response"] = self.create_guardrail_blocked_response(response=bedrock_guardrail_response) ######################################################### ########## 3. Add the guardrail to the applied guardrails header ########## @@ -1248,16 +1241,19 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ######################################################### ########## 1. Make the Bedrock API request ########## ######################################################### - bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = None - try: - bedrock_guardrail_response = await self.make_bedrock_api_request( - source="INPUT", - messages=filtered_messages, - request_data=data, - logging_event_type=GuardrailEventHooks.during_call, - ) - except GuardrailInterventionNormalStringError as e: - bedrock_guardrail_response = e.message + # A block with disable_exception_on_block=True raises ModifyResponseException + # from make_bedrock_api_request. Because during_call runs in an asyncio.gather + # alongside the LLM call (common_request_processing.py), swallowing the + # exception here to set data["mock_response"] was ineffective: route_request + # unpacked kwargs before this hook ran, and the LLM task's response was taken + # unconditionally. Letting the exception propagate cancels the LLM task and + # the endpoint handler returns the block response. + bedrock_guardrail_response = await self.make_bedrock_api_request( + source="INPUT", + messages=filtered_messages, + request_data=data, + logging_event_type=GuardrailEventHooks.during_call, + ) ######################################################### ######################################################### @@ -1272,8 +1268,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): updated_target_messages=updated_subset, target_indices=filter_result.target_indices, ) - if isinstance(bedrock_guardrail_response, str): - data["mock_response"] = self.create_guardrail_blocked_response(response=bedrock_guardrail_response) ######################################################### ########## 3. Add the guardrail to the applied guardrails header ########## @@ -1323,7 +1317,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # users should configure if they want input validation. Running an # extra INPUT scan here produced a duplicate post-call entry in the # trace and made no semantic sense for a "post-call" event. - output_content_bedrock: Optional[Union[BedrockGuardrailResponse, str]] = None + # A block with disable_exception_on_block=True raises ModifyResponseException + # from make_bedrock_api_request; that propagates to the endpoint handler, + # which returns a 200 whose message is the guardrail's blockedInputMessaging. + # Attach the LLM response to original_response so the synthetic block reply + # reports the real token usage the upstream call consumed instead of zero. try: output_content_bedrock = await self.make_bedrock_api_request( source="OUTPUT", @@ -1332,15 +1330,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): request_data=data, logging_event_type=GuardrailEventHooks.post_call, ) - except GuardrailInterventionNormalStringError as e: - output_content_bedrock = e.message + except ModifyResponseException as e: + if e.original_response is None: + e.original_response = response + raise ######################################################### ########## 2. Apply masking to response with output guardrail response ########## ######################################################### - if isinstance(output_content_bedrock, str): - response = self.create_guardrail_blocked_response(response=output_content_bedrock) - elif output_content_bedrock is not None: + if output_content_bedrock is not None: self._apply_masking_to_response( response=response, bedrock_guardrail_response=output_content_bedrock, @@ -1357,7 +1355,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): def _update_messages_with_updated_bedrock_guardrail_response( self, messages: List[AllMessageValues], - bedrock_guardrail_response: Union[BedrockGuardrailResponse, str], + bedrock_guardrail_response: BedrockGuardrailResponse, ) -> List[AllMessageValues]: """ Use the output from the bedrock guardrail to mask sensitive content in messages. @@ -1369,8 +1367,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): Returns: List of messages with content masked according to guardrail response """ - if isinstance(bedrock_guardrail_response, str): - return messages # Get masked texts from guardrail response masked_texts = self._extract_masked_texts_from_response(bedrock_guardrail_response) @@ -1422,7 +1418,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # pre_call / during_call. Bedrock will raise if the response # violates the guardrail policy. ################################################################### - output_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = None + # A block with disable_exception_on_block=True raises ModifyResponseException + # from make_bedrock_api_request. Non-streaming paths let it propagate so + # the endpoint handler turns it into a 200. Streaming can't do that: the + # SSE response headers are already flushed, so a raise would be serialized + # as an error frame by async_streaming_data_generator. Instead, replace + # the assembled response with the synthetic block content in-place and + # yield it as a normal stream, matching the shape a non-streaming block + # produces. try: output_guardrail_response = await self.make_bedrock_api_request( source="OUTPUT", @@ -1431,15 +1434,31 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): request_data=request_data, logging_event_type=GuardrailEventHooks.post_call, ) - except GuardrailInterventionNormalStringError as e: - output_guardrail_response = e.message + except ModifyResponseException as e: + # Preserve upstream usage from the LLM call we already + # consumed. Non-streaming blocks carry it via + # ModifyResponseException.original_response + + # _blocked_response_usage; streaming has to do the copy + # itself since the exception can't escape this generator. + _original_usage = getattr(assembled_model_response, "usage", None) + assembled_model_response = ModelResponse( + choices=[ + Choices( + index=0, + message=Message(role="assistant", content=e.message), + finish_reason="content_filter", + ) + ], + model=e.model, + ) + if _original_usage is not None: + assembled_model_response.usage = _original_usage + output_guardrail_response = None ######################################################################### ########## 2. Apply masking to response with output guardrail response ########## ######################################################################### - if isinstance(output_guardrail_response, str): - assembled_model_response = self.create_guardrail_blocked_response(response=output_guardrail_response) - elif output_guardrail_response is not None: + if output_guardrail_response is not None: self._apply_masking_to_response( response=assembled_model_response, bedrock_guardrail_response=output_guardrail_response, @@ -1732,13 +1751,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): inputs["texts"] = masked_texts return inputs - except (HTTPException, GuardrailInterventionNormalStringError): - # Let guardrail blocking exceptions propagate as-is so the proxy - # can return the correct HTTP status (400) or handle the - # GuardrailInterventionNormalStringError for disable_exception_on_block mode. - # Without this, the generic except below wraps them into a plain - # Exception, losing the HTTP semantics and preventing the proxy - # from properly blocking the call. + except (HTTPException, ModifyResponseException): + # Let guardrail blocking exceptions propagate as-is so the proxy can + # return the correct HTTP status (400 for HTTPException, 200 with the + # block message for ModifyResponseException in disable_exception_on_block + # mode). Without this, the generic except below wraps them into a plain + # Exception, losing the semantics and preventing the proxy from + # properly blocking the call. raise except Exception as e: verbose_proxy_logger.error("Bedrock Guardrail: Failed to apply guardrail: %s", str(e)) diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index 87c5e3bf2a9..f257b47404e 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -329,12 +329,13 @@ def test_bedrock_guardrail_filters_latest_user_message_when_enabled(): @pytest.mark.asyncio async def test_bedrock_apply_guardrail_blocked_with_disable_exception_on_block(): """ - Regression test for issue #20045: when disable_exception_on_block=True, - make_bedrock_api_request raises GuardrailInterventionNormalStringError. - apply_guardrail must let it propagate as-is so the proxy can handle it - properly instead of wrapping it in a generic Exception. + Regression test for LIT-4186: when disable_exception_on_block=True, a + Bedrock block raises ModifyResponseException. apply_guardrail must let it + propagate as-is so the endpoint handler (proxy_server.py) can turn it into + a 200 response with the block message as content, instead of the exception + surfacing as a bare 500. """ - from litellm.exceptions import GuardrailInterventionNormalStringError + from litellm.exceptions import ModifyResponseException guardrail = BedrockGuardrail( guardrail_name="test-bedrock-guard", @@ -346,18 +347,21 @@ async def test_bedrock_apply_guardrail_blocked_with_disable_exception_on_block() with patch.object( guardrail, "make_bedrock_api_request", new_callable=AsyncMock ) as mock_api: - mock_api.side_effect = GuardrailInterventionNormalStringError( - message="Sorry, your question in its current format is unable to be answered." + mock_api.side_effect = ModifyResponseException( + message="Sorry, your question in its current format is unable to be answered.", + model="bedrock-guardrail", + request_data={}, + guardrail_name="test-bedrock-guard", ) - with pytest.raises(GuardrailInterventionNormalStringError) as exc_info: + with pytest.raises(ModifyResponseException) as exc_info: await guardrail.apply_guardrail( inputs={"texts": ["harmful prompt content"]}, request_data={}, input_type="request", ) - assert "unable to be answered" in str(exc_info.value.message) + assert "unable to be answered" in exc_info.value.message @pytest.mark.asyncio diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index a23e89e576c..823ee05839f 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -1390,7 +1390,14 @@ async def test_bedrock_guardrail_disable_exception_on_block_non_streaming(): assert exception.status_code == 400 assert "Violated guardrail policy" in str(exception.detail) - # Test 2: disable_exception_on_block=True - should NOT raise exception + # Test 2: disable_exception_on_block=True - raises ModifyResponseException. + # LIT-4186: pre-fix, the native hook swallowed the block and set + # data["mock_response"], which was dead code (route_request already + # unpacked kwargs) so during_call let the model call proceed anyway. + # The correct contract is to raise ModifyResponseException so the endpoint + # handler returns a 200 with the block message as content. + from litellm.exceptions import ModifyResponseException + guardrail_disabled = BedrockGuardrail( guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT", @@ -1402,20 +1409,13 @@ async def test_bedrock_guardrail_disable_exception_on_block_non_streaming(): ) as mock_post: mock_post.return_value = mock_bedrock_response - # Should NOT raise exception when disable_exception_on_block=True - try: - response = await guardrail_disabled.async_moderation_hook( + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail_disabled.async_moderation_hook( data=request_data, user_api_key_dict=mock_user_api_key_dict, call_type="completion", ) - # Should succeed and return data (even though content was blocked) - assert response is not None - print("✅ No exception raised when disable_exception_on_block=True") - except Exception as e: - pytest.fail( - f"Should not raise exception when disable_exception_on_block=True, but got: {e}" - ) + assert exc_info.value.message == "I can't provide that information." @pytest.mark.asyncio @@ -1514,7 +1514,10 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): async for chunk in result_generator: pass - # Test 2: disable_exception_on_block=True - should NOT raise exception + # Test 2: disable_exception_on_block=True. Streaming can't raise up to the + # endpoint handler (SSE headers already flushed), so the block is delivered + # as a synthetic stream with finish_reason=content_filter and the block + # message as content -- same shape a non-streaming block produces. guardrail_disabled = BedrockGuardrail( guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT", @@ -1526,31 +1529,20 @@ async def test_bedrock_guardrail_disable_exception_on_block_streaming(): ) as mock_post: mock_post.return_value = mock_bedrock_response - # Should NOT raise exception when disable_exception_on_block=True - try: - result_generator = ( - guardrail_disabled.async_post_call_streaming_iterator_hook( - user_api_key_dict=mock_user_api_key_dict, - response=mock_streaming_response(), - request_data=request_data, - ) - ) - - # Consume the generator - should succeed without exceptions - result_chunks = [] - async for chunk in result_generator: - result_chunks.append(chunk) - - # Should have received chunks back even though content was blocked - assert len(result_chunks) > 0 - print( - "✅ Streaming completed without exception when disable_exception_on_block=True" - ) - - except Exception as e: - pytest.fail( - f"Should not raise exception when disable_exception_on_block=True in streaming, but got: {e}" - ) + result_generator = guardrail_disabled.async_post_call_streaming_iterator_hook( + user_api_key_dict=mock_user_api_key_dict, + response=mock_streaming_response(), + request_data=request_data, + ) + chunks = [c async for c in result_generator] + assert chunks, "streaming block should yield synthetic chunks, not empty" + assembled_content = "".join( + (c.choices[0].delta.content or "") + for c in chunks + if getattr(c, "choices", None) and getattr(c.choices[0], "delta", None) + ) + assert assembled_content == "I can't provide that information." + assert chunks[-1].choices[0].finish_reason == "content_filter" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index f43d8e85aca..bf237d8017b 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -2767,3 +2767,408 @@ async def test_grounding_output_blocked_raises_400(): ) assert exc_info.value.status_code == 400 + + +############################################################################### +# LIT-4186: disable_exception_on_block regression tests +# +# Before the fix, a Bedrock block with disable_exception_on_block=True raised +# GuardrailInterventionNormalStringError, which no proxy code handled: the +# unified pre_call path re-raised it, so the client saw HTTP 500 with the block +# message; the native during_call hook swallowed it and set data["mock_response"], +# which was dead code because route_request already unpacked kwargs. +# +# The fix converts blocks to ModifyResponseException at the raise site inside +# make_bedrock_api_request. That exception is already the industry-standard +# proxy contract (caught in proxy_server.py, anthropic_endpoints, etc.) and +# turns into a 200 response whose content is the block message. +############################################################################### + + +def _blocked_bedrock_httpx_response() -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{"text": "Sorry, the model cannot answer this question."}], + "assessments": [ + { + "topicPolicy": { + "topics": [{"name": "Denied", "type": "DENY", "action": "BLOCKED"}] + } + } + ], + } + return response + + +@pytest.mark.asyncio +async def test_make_bedrock_api_request_block_raises_modify_response_when_flag_set(): + from litellm.exceptions import ModifyResponseException + + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True, + ) + + request_data = {"model": "bedrock-nova-micro"} + mock_credentials = MagicMock() + mock_credentials.access_key = "k" + mock_credentials.secret_key = "s" + mock_credentials.token = None + + with ( + patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, + patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ): + mock_post.return_value = _blocked_bedrock_httpx_response() + + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail.make_bedrock_api_request( + source="INPUT", + messages=[{"role": "user", "content": "My name is John Doe"}], + request_data=request_data, + ) + + assert exc_info.value.message == "Sorry, the model cannot answer this question." + assert exc_info.value.model == "bedrock-nova-micro" + assert exc_info.value.guardrail_name == "test-bedrock-guard" + + +@pytest.mark.asyncio +async def test_make_bedrock_api_request_block_raises_http_400_when_flag_unset(): + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=False, + ) + + mock_credentials = MagicMock() + mock_credentials.access_key = "k" + mock_credentials.secret_key = "s" + mock_credentials.token = None + + with ( + patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, + patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ): + mock_post.return_value = _blocked_bedrock_httpx_response() + + with pytest.raises(HTTPException) as exc_info: + await guardrail.make_bedrock_api_request( + source="INPUT", + messages=[{"role": "user", "content": "hi"}], + request_data={"model": "bedrock-nova-micro"}, + ) + + assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_async_pre_call_hook_propagates_modify_response_on_block(): + """pre_call: block with disable_exception_on_block=True must raise + ModifyResponseException so the endpoint handler returns 200 with the block + message. Before LIT-4186 the exception was swallowed and only data + ["mock_response"] was mutated, which the unified pre_call path never read + (surfaced as HTTP 500).""" + from litellm.exceptions import ModifyResponseException + + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True, + ) + + request_data = { + "model": "bedrock-nova-micro", + "messages": [{"role": "user", "content": "My name is John Doe"}], + } + mock_credentials = MagicMock() + mock_credentials.access_key = "k" + mock_credentials.secret_key = "s" + mock_credentials.token = None + + with ( + patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, + patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ): + mock_post.return_value = _blocked_bedrock_httpx_response() + + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=request_data, + call_type="acompletion", + ) + + assert exc_info.value.message == "Sorry, the model cannot answer this question." + # No `mock_response` mutation: the old broken contract must be gone + # (route_request unpacks kwargs before this hook runs, so `mock_response` + # would never reach the LLM call anyway). + assert "mock_response" not in request_data + + +@pytest.mark.asyncio +async def test_async_moderation_hook_propagates_modify_response_on_block(): + """during_call: block must raise ModifyResponseException from the moderation + task so the surrounding asyncio.gather cancels the LLM call, instead of + the old behavior of swallowing the block and letting the model call proceed + (LIT-4186 symptom 2: silent bypass, model billed).""" + from litellm.exceptions import ModifyResponseException + + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True, + ) + + request_data = { + "model": "bedrock-nova-micro", + "messages": [{"role": "user", "content": "My name is John Doe"}], + } + mock_credentials = MagicMock() + mock_credentials.access_key = "k" + mock_credentials.secret_key = "s" + mock_credentials.token = None + + with ( + patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, + patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ): + mock_post.return_value = _blocked_bedrock_httpx_response() + + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=UserAPIKeyAuth(), + call_type="acompletion", + ) + + assert exc_info.value.message == "Sorry, the model cannot answer this question." + + +@pytest.mark.asyncio +async def test_async_post_call_success_hook_attaches_original_response_on_block(): + """post_call: block must raise ModifyResponseException and attach the LLM + response to `original_response` so the synthetic block reply reports the + upstream call's real token usage instead of zero.""" + from litellm.exceptions import ModifyResponseException + + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True, + ) + + request_data = { + "model": "bedrock-nova-micro", + "messages": [{"role": "user", "content": "hi"}], + } + llm_response = _model_response("Hello John Doe! The capital of France is Paris.") + mock_credentials = MagicMock() + mock_credentials.access_key = "k" + mock_credentials.secret_key = "s" + mock_credentials.token = None + + with ( + patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, + patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ): + mock_post.return_value = _blocked_bedrock_httpx_response() + + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail.async_post_call_success_hook( + data=request_data, + user_api_key_dict=UserAPIKeyAuth(), + response=llm_response, + ) + + assert exc_info.value.original_response is llm_response + + +@pytest.mark.asyncio +async def test_apply_guardrail_propagates_modify_response_on_block(): + """apply_guardrail (unified path used by pre_call / /apply_guardrail + endpoint) must let ModifyResponseException propagate as-is so the endpoint + handler catches it and returns a 200.""" + from litellm.exceptions import ModifyResponseException + + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True, + ) + + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api: + mock_api.side_effect = ModifyResponseException( + message="Sorry, the model cannot answer this question.", + model="bedrock-nova-micro", + request_data={}, + guardrail_name="test-bedrock-guard", + ) + + with pytest.raises(ModifyResponseException) as exc_info: + await guardrail.apply_guardrail( + inputs={"texts": ["My name is John Doe"]}, + request_data={"model": "bedrock-nova-micro"}, + input_type="request", + ) + + assert exc_info.value.message == "Sorry, the model cannot answer this question." + + +@pytest.mark.asyncio +async def test_streaming_post_call_block_yields_synthetic_stream_not_raise(): + """LIT-4186 regression: with disable_exception_on_block=True, streaming + post_call blocks must be delivered as a synthetic stream (finish_reason= + content_filter, block message as content), NOT raised. Pre-fix the local + handler already produced this shape; the LIT-4186 refactor briefly turned + it into an SSE 500 by letting ModifyResponseException escape the streaming + generator. This test locks in the correct streaming contract. + """ + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True, + ) + + async def _stream(): + yield ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content="Coffee is a popular"), + ) + ] + ) + yield ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=" beverage."), finish_reason="stop")] + ) + + mock_credentials = MagicMock() + mock_credentials.access_key = "k" + mock_credentials.secret_key = "s" + mock_credentials.token = None + + with ( + patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post, + patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ): + mock_post.return_value = _blocked_bedrock_httpx_response() + + chunks = [ + c + async for c in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_stream(), + request_data={"model": "bedrock-nova-micro"}, + ) + ] + + assert chunks, "streaming block should yield synthetic chunks, not error out" + assembled_content = "".join( + (c.choices[0].delta.content or "") + for c in chunks + if getattr(c, "choices", None) and getattr(c.choices[0], "delta", None) + ) + assert assembled_content == "Sorry, the model cannot answer this question." + assert chunks[-1].choices[0].finish_reason == "content_filter" + + +@pytest.mark.asyncio +async def test_streaming_post_call_block_preserves_upstream_usage(): + """LIT-4186: streaming block must report the usage the upstream LLM call + actually consumed. Non-streaming blocks carry it via original_response + + _blocked_response_usage in the endpoint handler; streaming has to copy it + onto the synthetic ModelResponse directly since the exception can't escape + the SSE generator. Without this, clients see accurate billing on + non-streaming blocks and zero on streaming blocks -- silent revenue leak.""" + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage + + guardrail = BedrockGuardrail( + guardrail_name="test-bedrock-guard", + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + disable_exception_on_block=True, + ) + + async def _stream_with_usage(): + # Terminal chunk carrying usage, as OpenAI-style streams do with + # stream_options={"include_usage": True}. stream_chunk_builder + # aggregates this into the assembled ModelResponse's .usage. + yield ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="Coffee is delicious"))] + ) + yield ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + usage=Usage(prompt_tokens=42, completion_tokens=17, total_tokens=59), + ) + + mock_credentials = MagicMock() + mock_credentials.access_key = "k" + mock_credentials.secret_key = "s" + mock_credentials.token = None + + with ( + patch.object(guardrail.async_handler, "post", new_callable=AsyncMock) as mock_post, + patch.object(guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")), + patch.object(guardrail, "_prepare_request", return_value=MagicMock()), + ): + mock_post.return_value = _blocked_bedrock_httpx_response() + + chunks = [ + c + async for c in guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_stream_with_usage(), + request_data={"model": "bedrock-nova-micro"}, + ) + ] + + # Find the chunk carrying usage (MockResponseIterator emits it on the + # terminating chunk when the source ModelResponse has .usage set) + usage_chunks = [c for c in chunks if getattr(c, "usage", None) is not None] + assert usage_chunks, "streaming block should carry the upstream call's usage on at least one chunk" + reported_usage = usage_chunks[-1].usage + assert reported_usage.prompt_tokens == 42 + assert reported_usage.completion_tokens == 17 + assert reported_usage.total_tokens == 59 From bff2c952e0339a736f451797974f507e6ccbda48 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 13:19:56 -0700 Subject: [PATCH 067/101] fix(ui): key the edit form's browser-held token handling off the effective auth type The edit form decided auth mode from the saved mcpServer.auth_type in fetchTools while the authorize flow used the current form value, so a token authorized after switching the form to a client-forwarded mode was never forwarded as the x-mcp header until the server was saved. A shared getEffectiveAuthType (form value falling back to the saved record) is now the single decision point for token receipt and tool loading The save path classified the staged token with getMcpOAuthMode, which returns null for true_passthrough and oauth_delegate, so the staged token was dropped on save instead of being committed to sessionStorage the way the create form's submit path does. The passthrough branch now also covers the client-forwarded modes; the token still never enters the server row --- .../mcp_tools/mcp_server_edit.test.tsx | 70 +++++++++++++++++++ .../components/mcp_tools/mcp_server_edit.tsx | 17 +++-- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index 0579a3208d1..d55f993b926 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -1173,6 +1173,76 @@ describe("MCPServerEdit (OAuth token persistence on save)", () => { expect(onSuccess).not.toHaveBeenCalled(); }); + it.each([["true_passthrough"], ["oauth_delegate"]])( + "persists the staged token to sessionStorage on save for the %s mode", + async (authType) => { + // Regression: the save path classified the staged token with getMcpOAuthMode, which returns + // null for the client-forwarded modes, so setToken was never called and the browser-held + // token was dropped on save; the create form's submit path already committed it. + mockOauth.tokenResponse = { access_token: "cf-tok", expires_in: 1800, token_type: "bearer" }; + vi.mocked(networking.updateMCPServer).mockResolvedValue({ + ...interactiveOAuthServer, + auth_type: authType, + }); + + render( + , + ); + + await act(async () => { + fireEvent.click(screen.getAllByRole("button", { name: "Save Changes" })[0]); + }); + + await waitFor(() => { + expect(mockSetToken).toHaveBeenCalledWith( + "oauth_server_1", + expect.objectContaining({ access_token: "cf-tok" }), + "user-1", + ); + }); + expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled(); + const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; + expect(payload.credentials).toBeUndefined(); + }, + ); + + it("forwards a newly authorized browser-held token for tool loading before the form is saved", async () => { + // Regression: fetchTools keyed the browser-held decision off the saved mcpServer.auth_type, so + // after switching the form to true_passthrough and authorizing, the fresh token was not sent as + // the x-mcp header until the server was saved. + vi.mocked(networking.listMCPTools).mockResolvedValue({ tools: [], error: null }); + mockIsTokenValid.mockReturnValue(false); + + render( + , + ); + + await selectAntOption("Authentication", "OAuth Delegate (client-supplied upstream token)"); + mockOauth.tokenResponse = { access_token: "fresh-tok", token_type: "bearer" }; + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + await waitFor(() => { + const withHeaders = vi + .mocked(networking.listMCPTools) + .mock.calls.find(([, , headers]) => headers && JSON.stringify(headers).includes("fresh-tok")); + expect(withHeaders).toBeTruthy(); + }); + }); + it("persists the passthrough token to sessionStorage on save after authorize", async () => { mockOauth.tokenResponse = { access_token: "pt-tok", expires_in: 1800, token_type: "bearer" }; vi.mocked(networking.updateMCPServer).mockResolvedValue({ diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 59d7b28aacb..ee7938d2904 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -131,6 +131,11 @@ const MCPServerEdit: React.FC = ({ } }; + // The auth mode every decision must key off: the admin's in-flight form selection wins over the + // saved record, so authorizing, loading tools, and saving all agree with what the form shows. Paths + // that read only mcpServer.auth_type go stale the moment the admin switches modes in the form. + const getEffectiveAuthType = () => form.getFieldValue("auth_type") ?? mcpServer.auth_type; + const { startOAuthFlow, status: oauthStatus, @@ -178,8 +183,7 @@ const MCPServerEdit: React.FC = ({ return; } - const effectiveAuthType = form.getFieldValue("auth_type") ?? mcpServer.auth_type; - if (isClientForwardedTokenMode(effectiveAuthType)) { + if (isClientForwardedTokenMode(getEffectiveAuthType())) { const browserHeldToken = { access_token: token.access_token, expires_in: token.expires_in, @@ -388,7 +392,7 @@ const MCPServerEdit: React.FC = ({ oauth2_flow: mcpServer.oauth2_flow, delegate_auth_to_upstream: mcpServer.delegate_auth_to_upstream, }) === "passthrough"; - const isBrowserHeldTokenMode = isClientForwardedTokenMode(mcpServer.auth_type); + const isBrowserHeldTokenMode = isClientForwardedTokenMode(getEffectiveAuthType()); if (isPassthrough || isBrowserHeldTokenMode) { const token = oauthTokenResponse?.access_token ?? @@ -749,8 +753,9 @@ const MCPServerEdit: React.FC = ({ const updated = await updateMCPServer(accessToken, payload); // Persist the token staged via "Authorize & Fetch" (mirrors the create flow's - // commit-on-submit): OBO writes the per-user token to the DB, passthrough keeps - // it in sessionStorage. M2M/static auth resolve server-side and need neither. + // commit-on-submit): OBO writes the per-user token to the DB; legacy passthrough and the + // client-forwarded modes (true_passthrough / oauth_delegate) keep it in sessionStorage and + // never in the server row. M2M/static auth resolve server-side and need neither. if (oauthTokenResponse?.access_token) { const oauthMode = getMcpOAuthMode({ auth_type: restValues.auth_type, @@ -767,7 +772,7 @@ const MCPServerEdit: React.FC = ({ scopes: typeof scope === "string" && scope ? scope.split(" ") : undefined, }; await storeMCPOAuthUserCredential(accessToken, mcpServer.server_id, oauthCredentialPayload); - } else if (oauthMode === "passthrough") { + } else if (oauthMode === "passthrough" || isClientForwardedTokenMode(restValues.auth_type)) { const browserHeldToken = { access_token: oauthTokenResponse.access_token, expires_in: oauthTokenResponse.expires_in, From 97d09512969f2adf5b39863cc18e9a75b0915d78 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 9 Jul 2026 16:39:47 -0400 Subject: [PATCH 068/101] test(e2e): make dynamic model provisioning robust on split deployments (#32670) create_model now waits until the new deployment is servable on the data plane (polls /v1/models) before returning, instead of assuming /model/new makes it instantly callable. On a split control/data-plane proxy the gateway only sees a model after its next DB reload, so an immediate call raced the reload and 400'd with "Invalid model name passed" (embeddings, responses, messages, ocr, ...). It also stops pinning model_info.id to the model_name, letting the proxy assign a unique model_id. Re-registering a fixed-name deployment (the batch suite's openai-batch et al.) after a failed teardown no longer collides on the model_id unique constraint (prisma UniqueViolationError surfaced as the generic 500 "Failed to add model to db", erroring every batch_lifecycle case at setup) --- tests/e2e/e2e_gateway.py | 49 +++++++++++++++++++++++--- tests/e2e/models.py | 18 +++++++++- tests/e2e/test_e2e_gateway.py | 66 ++++++++++++++++++++++++++++++++--- 3 files changed, 122 insertions(+), 11 deletions(-) diff --git a/tests/e2e/e2e_gateway.py b/tests/e2e/e2e_gateway.py index 055d06d1c79..d62c9c4b17b 100644 --- a/tests/e2e/e2e_gateway.py +++ b/tests/e2e/e2e_gateway.py @@ -42,6 +42,7 @@ from models import ( ModelMode, ModelNewBody, ModelNewResponse, + ModelsListResponse, OcrBody, OcrResponse, SpendLogRow, @@ -125,21 +126,59 @@ class Gateway: litellm_params: LiteLLMParamsBody, mode: ModelMode | None = None, ) -> str: - """Register a deployment under `model_name` (id == model_name) and return the - model_id. add_deployment runs synchronously in /model/new, so the model is - callable as soon as this returns.""" - return unwrap( + """Register a deployment under `model_name` and return its proxy-assigned + model_id, once the model is actually servable on the data plane. + + /model/new is a control-plane route; in a split control/data-plane + deployment the gateway (data plane, which serves /chat, /ocr, ...) only + picks the new model up on its next DB reload, so a call issued the instant + this returns can race the reload and 400 with "Invalid model name passed". + We therefore poll the data-plane /v1/models until the model appears before + handing back, so callers can invoke it immediately. In the monolithic case + it is already present on the first poll, so this adds one request.""" + model_id = unwrap( self.transport.post( "/model/new", headers=self.transport.master, json=ModelNewBody( model_name=model_name, litellm_params=litellm_params, - model_info=ModelInfoBody(id=model_name, mode=mode), + model_info=ModelInfoBody(mode=mode), ), response_type=ModelNewResponse, ) ).model_id + self._await_model_servable(model_name) + return model_id + + def _await_model_servable(self, model_name: str) -> None: + """Block until the data plane lists `model_name`, or fail loudly if it does + not within poll_timeout (a real propagation/config problem, surfaced here + instead of as a downstream "Invalid model name passed").""" + deadline = time.monotonic() + self.poll_timeout + last_result: Result[ModelsListResponse] | None = None + while time.monotonic() < deadline: + last_result = self.transport.get( + "/v1/models", + headers=self.transport.master, + params=NoBody(), + response_type=ModelsListResponse, + ) + if isinstance(last_result, Success) and any( + entry.id == model_name for entry in last_result.data.data + ): + return + time.sleep(self.poll_interval) + last_error = ( + f"; last /v1/models poll did not succeed: {last_result}" + if last_result is not None and not isinstance(last_result, Success) + else "" + ) + raise AssertionError( + f"model {model_name!r} was created but never became servable on the data " + f"plane within {self.poll_timeout}s of /model/new (control/data-plane " + f"propagation or STORE_MODEL_IN_DB reload issue){last_error}" + ) def delete_model(self, model_id: str) -> None: result = self.transport.post( diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 0490db286ea..f287058b313 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -394,7 +394,11 @@ ModelMode = Literal["batch", "realtime", "image_generation"] class ModelInfoBody(BaseModel): - id: str + # id is left unset so the proxy assigns a unique model_id per deployment. + # Pinning it to the model_name made re-registrations of a fixed-name model + # (e.g. the batch suite's openai-batch) collide on the model_id unique + # constraint when a prior run's teardown had not removed the row. + id: str | None = None mode: ModelMode | None = None @@ -410,6 +414,18 @@ class ModelNewResponse(BaseModel): model_id: str +class ModelListEntry(BaseModel): + id: str + + +class ModelsListResponse(BaseModel): + """GET /v1/models on the data plane: the deployments the gateway can actually + serve right now. Used to confirm a freshly created model has propagated from + the control plane before a test calls it.""" + + data: tuple[ModelListEntry, ...] = () + + class ModelDeleteBody(BaseModel): id: str diff --git a/tests/e2e/test_e2e_gateway.py b/tests/e2e/test_e2e_gateway.py index a6dcc6112d6..9a9aa2fd2cc 100644 --- a/tests/e2e/test_e2e_gateway.py +++ b/tests/e2e/test_e2e_gateway.py @@ -10,6 +10,7 @@ signature drift fails here instead of in a live stage run. from dataclasses import dataclass, field +import pytest from pydantic import BaseModel from batches.batch_client import BatchClient @@ -21,26 +22,38 @@ from e2e_http import ( Result, StreamingResponse, Success, + UnknownApiError, ) from models import ( LiteLLMParamsBody, ModelDeleteBody, ModelNewBody, ModelNewResponse, + ModelsListResponse, ) @dataclass class _RecordingTransport: """Typed fake fulfilling the Transport protocol; records every post and - answers with a canned success so the test asserts on what was sent.""" + answers with a canned success so the test asserts on what was sent. + + `get("/v1/models")` reports a created model as servable only after + `servable_after_gets` polls, so a test can drive the data-plane wait in + create_model.""" posts: list[tuple[str, BaseModel]] = field(default_factory=list) + servable_after_gets: int = 0 + models_error: UnknownApiError | None = None + model_gets: int = 0 + _created: list[str] = field(default_factory=list) def post[R: BaseModel]( self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] ) -> Result[R]: self.posts.append((path, json)) + if path == "/model/new" and isinstance(json, ModelNewBody): + self._created.append(json.model_name) payload = ( {"model_id": "registered-id"} if response_type is ModelNewResponse else {} ) @@ -70,7 +83,15 @@ class _RecordingTransport: params: BaseModel, response_type: type[R], ) -> Result[R]: - raise AssertionError("get is not part of model management") + if path == "/v1/models" and response_type is ModelsListResponse: + self.model_gets += 1 + if self.models_error is not None: + return self.models_error + visible = self._created if self.model_gets > self.servable_after_gets else [] + return Success( + data=response_type.model_validate({"data": [{"id": name} for name in visible]}) + ) + raise AssertionError(f"unexpected get: {path}") def delete[R: BaseModel]( self, path: str, *, headers: BaseModel, json: BaseModel, response_type: type[R] @@ -106,7 +127,7 @@ class _RecordingTransport: def test_gateway_create_model_registers_deployment_and_returns_model_id() -> None: transport = _RecordingTransport() - gateway = Gateway(transport=transport) + gateway = Gateway(transport=transport, poll_interval=0.0) model_id = gateway.create_model( "e2e-test-model", LiteLLMParamsBody(model="openai/gpt-4o-mini") @@ -117,13 +138,48 @@ def test_gateway_create_model_registers_deployment_and_returns_model_id() -> Non assert path == "/model/new" assert isinstance(body, ModelNewBody) assert body.model_name == "e2e-test-model" - assert body.model_info.id == "e2e-test-model" + # No pinned model_id: the proxy assigns a unique one, so a fixed-name model + # re-registered after a failed teardown can't collide on the id constraint. + assert body.model_info.id is None assert body.model_info.mode is None + # It confirmed data-plane visibility before returning. + assert transport.model_gets >= 1 + + +def test_gateway_create_model_waits_until_servable_on_the_data_plane() -> None: + # The model shows up on /v1/models only on the third poll (simulating the + # gateway's delayed DB reload in a split deployment); create_model must keep + # polling instead of returning after /model/new. + transport = _RecordingTransport(servable_after_gets=2) + gateway = Gateway(transport=transport, poll_interval=0.0) + + gateway.create_model("e2e-late-model", LiteLLMParamsBody(model="openai/gpt-4o-mini")) + + assert transport.model_gets == 3 + + +def test_gateway_create_model_fails_loudly_when_never_servable() -> None: + transport = _RecordingTransport(servable_after_gets=10**9) + gateway = Gateway(transport=transport, poll_timeout=0.05, poll_interval=0.0) + + with pytest.raises(AssertionError, match="never became servable"): + gateway.create_model("e2e-ghost-model", LiteLLMParamsBody(model="openai/gpt-4o-mini")) + + +def test_gateway_create_model_surfaces_the_last_data_plane_error() -> None: + transport = _RecordingTransport( + models_error=UnknownApiError(status_code=503, body="data plane down") + ) + gateway = Gateway(transport=transport, poll_timeout=0.05, poll_interval=0.0) + + with pytest.raises(AssertionError, match="data plane down") as excinfo: + gateway.create_model("e2e-flaky-model", LiteLLMParamsBody(model="openai/gpt-4o-mini")) + assert "503" in str(excinfo.value) def test_batch_client_create_model_registers_a_batch_mode_deployment() -> None: transport = _RecordingTransport() - client = BatchClient(gateway=Gateway(transport=transport)) + client = BatchClient(gateway=Gateway(transport=transport, poll_interval=0.0)) model_id = client.create_model( "e2e-batch-model", LiteLLMParamsBody(model="openai/gpt-4o-mini") From d1a79f79713d700d2b685b76ae2c262853bb1fa7 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 9 Jul 2026 13:48:20 -0700 Subject: [PATCH 069/101] fix(ui): rename Virtual Keys 'Key Hash' filter label to 'Key ID' (#32672) --- .../src/components/VirtualKeysPage/VirtualKeysTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index 00f4304c8a9..cae6dc54df5 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -578,7 +578,7 @@ export function VirtualKeysTable() { }, { name: "Key Hash", - label: "Key Hash", + label: "Key ID", isSearchable: false, }, ]; From 5cf269088cca64f9fa16faeba4dedd00bf48486d Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 9 Jul 2026 13:48:47 -0700 Subject: [PATCH 070/101] fix(proxy): capture logging_obj before post_call_failure_hook pops it in ModifyResponseException streaming path (#32665) * fix(guardrails/bedrock): honor disable_exception_on_block by raising ModifyResponseException The Bedrock-specific GuardrailInterventionNormalStringError predates the unified guardrails refactor and no proxy code path handles it, so a block with the flag set surfaced as an uncaught Exception -> HTTP 500 in pre_call mode and was silently discarded in during_call mode (model call proceeded in the parallel asyncio.gather; the block hook's data["mock_response"] mutation happened after route_request had already unpacked kwargs). Convert the block to ModifyResponseException at the raise site inside make_bedrock_api_request. That exception is the industry-standard proxy contract already caught in proxy_server, anthropic_endpoints, response_api _endpoints, and pass_through_endpoints; it turns into a 200 response with finish_reason=content_filter and the block message as content, which is exactly what the flag was documented to yield. Post-call blocks attach the LLM response to original_response so the synthetic reply reports the upstream call's real token usage instead of zero. Deletes the now-orphaned GuardrailInterventionNormalStringError class and the dead create_guardrail_blocked_response / mock_response plumbing in the Bedrock hooks; updates the existing tests that had locked in the buggy contract. Resolves LIT-4186 * chore(guardrails/bedrock): drop dead str branch in _update_messages_with_updated_bedrock_guardrail_response Follow-up to the disable_exception_on_block fix. That method used to receive either a BedrockGuardrailResponse or a plain string (the block message, when the flag was set). Now that a block always raises ModifyResponseException before this method runs, the string branch is unreachable; tighten the type to BedrockGuardrailResponse and delete the guard. * fix(guardrails/bedrock): streaming post_call block yields synthetic stream instead of surfacing as SSE 500 Regression from the LIT-4186 refactor: pre-refactor, the streaming post_call iterator caught GuardrailInterventionNormalStringError locally and replaced the assembled response with a synthetic content-filter message, then re-emitted it as chunks via MockResponseIterator. After the refactor the exception was re-raised as ModifyResponseException, which async_streaming_data_generator serializes as a proxy 500 error frame because the SSE response headers are already flushed by the time the block fires. Non-streaming paths still let ModifyResponseException propagate to the endpoint handler (which converts it into a 200). Streaming can't do that, so keep the local synthesis: on the exception, rebind the assembled response to a ModelResponse whose single choice carries the block message as content and finish_reason=content_filter, and let the downstream MockResponseIterator emit it as chunks. Same shape a non-streaming block produces. Adds a mapped-file regression test that mutation-kills the raise behavior and locks in the synthetic-stream contract. * fix(guardrails/bedrock): preserve upstream usage on streaming post_call block Non-streaming post_call blocks report the upstream LLM call's real token usage via ModifyResponseException.original_response, which the endpoint handler unwraps through _blocked_response_usage. Streaming post_call synthesizes its own ModelResponse locally (the exception can't escape the SSE generator), and previously left .usage unset, so the client saw accurate billing on non-streaming blocks and zero on streaming blocks -- silent revenue leak. Copy the assembled response's .usage onto the synthetic block response before yielding. Pre-refactor code had the same gap (create_guardrail_blocked_response never set usage); this is a net improvement, not a regression fix. * fix(proxy): capture logging_obj before post_call_failure_hook pops it in ModifyResponseException streaming path post_call_failure_hook removes litellm_logging_obj from request_data before iterating callbacks (it's not serialisable). The streaming branch of the ModifyResponseException handler read it from _data after that call, so it always received None and CustomStreamWrapper.__init__ crashed with AttributeError: NoneType has no attribute model_call_details. Capture it before the hook runs so the streaming path gets a valid object. Co-authored-by: Mateo Wang * test(proxy): add regression for streaming ModifyResponseException logging_obj capture Covers the bug where logging_obj was read from request_data after post_call_failure_hook had already popped it, causing CustomStreamWrapper to crash with AttributeError. Co-authored-by: Mateo Wang * test(proxy): drive real chat_completion in ModifyResponseException streaming logging_obj regression The original test inlined the fix pattern (capture before pop) in its own body rather than calling the actual chat_completion handler in proxy_server.py, so a revert of the fix left the test passing. Confirmed via mutation check: reverting the two-line source fix and re-running left the test green. Rewrite the test to drive chat_completion directly: - patch _read_request_body so chat_completion sees the seeded dict - patch ProxyBaseLLMRequestProcessing.base_process_llm_request to raise ModifyResponseException with the same request_data - patch proxy_logging_obj so post_call_failure_hook mutates the dict the way production does (pops litellm_logging_obj) - intercept CustomStreamWrapper.__init__ and assert logging_obj is the non-None object seeded in request_data Mutation-verified: reverting the source fix now surfaces the exact production crash inside CustomStreamWrapper's __init__ (AttributeError: NoneType has no attribute model_call_details) rather than a silently-passing test. Addresses Greptile P1 on PR #32665. --------- Co-authored-by: Mateo Wang --- litellm/proxy/proxy_server.py | 4 +- .../test_bedrock_guardrails.py | 102 ++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 13e2d4f1252..4114bda47c9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8544,6 +8544,8 @@ async def chat_completion( except ModifyResponseException as e: # Guardrail flagged content in passthrough mode - return 200 with violation message _data = e.request_data + # Capture logging_obj before post_call_failure_hook pops it from _data. + _logging_obj = _data.get("litellm_logging_obj") await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, @@ -8563,7 +8565,7 @@ async def chat_completion( completion_stream=_iterator, model=e.model, custom_llm_provider="cached_response", - logging_obj=_data.get("litellm_logging_obj", None), + logging_obj=_logging_obj, ) selected_data_generator = select_data_generator( response=_streaming_response, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index bf237d8017b..15827b80bcf 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -3172,3 +3172,105 @@ async def test_streaming_post_call_block_preserves_upstream_usage(): assert reported_usage.prompt_tokens == 42 assert reported_usage.completion_tokens == 17 assert reported_usage.total_tokens == 59 + + +############################################################################### +# Regression test for the streaming logging_obj bug found during live testing. +# +# post_call_failure_hook (proxy_server.py) pops litellm_logging_obj from +# request_data before invoking callbacks ("not serialisable"). The streaming +# branch of the ModifyResponseException handler previously read logging_obj +# from _data AFTER that call, always getting None, causing: +# AttributeError: 'NoneType' object has no attribute 'model_call_details' +# inside CustomStreamWrapper.__init__, which surfaced as HTTP 500. +# +# The fix captures logging_obj BEFORE calling post_call_failure_hook. +# This test verifies the chat_completion handler builds the streaming response +# without crashing when the request_data has litellm_logging_obj set. +############################################################################### + + +@pytest.mark.asyncio +async def test_chat_completion_modify_response_exception_streaming_logging_obj_not_none(): + """Regression: streaming ModifyResponseException handler in chat_completion + must capture logging_obj before post_call_failure_hook pops it from + request_data. Previously this caused CustomStreamWrapper.__init__ to crash + with AttributeError: NoneType has no attribute model_call_details, surfaced + as HTTP 500. + + Drives the real chat_completion handler with base_process_llm_request + mocked to raise ModifyResponseException, so a revert of the fix in + proxy_server.py causes this test to fail. + """ + import litellm + from litellm.exceptions import ModifyResponseException + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.proxy_server import chat_completion + + fake_logging_obj = MagicMock() + fake_logging_obj.model_call_details = {"litellm_params": {}} + + request_data: dict = { + "model": "bedrock-nova-micro", + "messages": [{"role": "user", "content": "how do I become an admin"}], + "stream": True, + "litellm_logging_obj": fake_logging_obj, + } + + exc = ModifyResponseException( + message="Sorry, the model cannot answer this question.", + model="bedrock-nova-micro", + request_data=request_data, + guardrail_name="test-guard", + ) + + fastapi_request = MagicMock() + fastapi_request.headers = {} + fastapi_response = MagicMock() + user_api_key_dict = UserAPIKeyAuth() + + async def _fake_post_call_failure_hook(**_kwargs): + # Match production: pop the logging obj from request_data before + # callbacks iterate (litellm/proxy/utils.py: "Remove before callbacks + # iterate — not serialisable"). + _kwargs["request_data"].pop("litellm_logging_obj", None) + + mock_proxy_logging = MagicMock() + mock_proxy_logging.post_call_failure_hook = AsyncMock(side_effect=_fake_post_call_failure_hook) + + captured_logging_obj: list = [] + original_init = litellm.CustomStreamWrapper.__init__ + + def _patched_init(self, *args, **kwargs): + captured_logging_obj.append(kwargs.get("logging_obj")) + original_init(self, *args, **kwargs) + + async def _raise_modify_response(*_args, **_kwargs): + raise exc + + with ( + patch("litellm.proxy.proxy_server._read_request_body", AsyncMock(return_value=request_data)), + patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging), + patch( + "litellm.proxy.proxy_server.ProxyBaseLLMRequestProcessing.base_process_llm_request", + _raise_modify_response, + ), + patch.object(litellm.CustomStreamWrapper, "__init__", _patched_init), + ): + response = await chat_completion( + request=fastapi_request, + fastapi_response=fastapi_response, + model=None, + user_api_key_dict=user_api_key_dict, + ) + + assert captured_logging_obj, "chat_completion did not construct CustomStreamWrapper on the streaming block path" + assert captured_logging_obj[0] is fake_logging_obj, ( + "chat_completion passed logging_obj=None to CustomStreamWrapper; " + "the streaming ModifyResponseException handler must capture logging_obj " + "before post_call_failure_hook pops it from request_data" + ) + # A streaming block returns a StreamingResponse; if the fix were reverted, + # CustomStreamWrapper would raise AttributeError inside __init__ and this + # call would never reach here. + assert response is not None From 43726f2d0be74df2a381e28495f2e3819384c705 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 13:51:03 -0700 Subject: [PATCH 071/101] refactor(ui): useTestMCPConnection uses the shared isClientForwardedTokenMode helper The helper extraction missed this call site, leaving an inline duplicate of the two-mode check that could drift from the shared definition --- ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx b/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx index 3208b6b02b2..d27e1ca8bf9 100644 --- a/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx +++ b/ui/litellm-dashboard/src/hooks/useTestMCPConnection.tsx @@ -1,6 +1,6 @@ import { useState, useEffect, useCallback } from "react"; import { testMCPToolsListRequest } from "../components/networking"; -import { AUTH_TYPE, OAUTH_FLOW, TRANSPORT } from "@/components/mcp_tools/types"; +import { AUTH_TYPE, OAUTH_FLOW, TRANSPORT, isClientForwardedTokenMode } from "@/components/mcp_tools/types"; interface MCPServerConfig { server_id?: string; @@ -56,8 +56,7 @@ export const useTestMCPConnection = ({ // Check if we have the minimum required fields to fetch tools const isM2MOAuth = formValues.auth_type === AUTH_TYPE.OAUTH2 && formValues.oauth_flow_type === OAUTH_FLOW.M2M; - const isBrowserHeldTokenMode = - formValues.auth_type === AUTH_TYPE.TRUE_PASSTHROUGH || formValues.auth_type === AUTH_TYPE.OAUTH_DELEGATE; + const isBrowserHeldTokenMode = isClientForwardedTokenMode(formValues.auth_type); const requiresOAuthToken = (formValues.auth_type === AUTH_TYPE.OAUTH2 && !isM2MOAuth) || isBrowserHeldTokenMode; const isOpenAPITransport = formValues.transport === TRANSPORT.OPENAPI; const hasEndpoint = isOpenAPITransport ? !!formValues.spec_path : !!formValues.url; From 8519d7fc24973457fc66e6cd25a504af6f1b8208 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 9 Jul 2026 16:54:45 -0400 Subject: [PATCH 072/101] test: litellm fix failing tests (#32577) * fix: rust ocr tests finally pass * fix: move realtime dir * fix(realtime): normalize azure realtime api_base to host for Foundry endpoints The azure realtime handler appended the realtime path to api_base verbatim, so a Foundry base carrying a project path (.../api/projects/) produced an invalid realtime URL and the websocket handshake hung. Normalize api_base to scheme and host before building the realtime path so both Azure OpenAI and Foundry bases connect Point the e2e realtime azure deployment at the GA gpt-realtime model and stop passing the os.environ refs the realtime path never unwraps, resolving them from the gateway env by name instead. Drop the local docker-compose scaffolding from the tree * test(e2e): add Gateway.list_files and list_fine_tuning_jobs for the discovery suite The discovery endpoints suite calls client.gateway.list_files and list_fine_tuning_jobs, which did not exist on Gateway, so both tests errored with AttributeError before reaching the proxy. Add the two GET wrappers using the existing FileListResponse / FineTuningJobsResponse models * revert(realtime): drop azure realtime api_base host-normalization The azure realtime handshake failure was a config issue, not a litellm bug: the realtime base was set to the Azure AI Foundry project endpoint (.../api/projects/

), but the OpenAI-compatible realtime route lives at the resource root. litellm correctly appends the realtime path to whatever base it is given, so pointing the realtime deployment at the resource root is the fix and no core change is needed * fix(ocr): route azure_ai doc-intelligence to its own endpoint at the source get_llm_provider inherits AZURE_AI_API_BASE into api_base for every azure_ai/* OCR model, but Azure Document Intelligence is a separate resource reached via AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT, so doc-intelligence requests went to the wrong host. Stop inheriting the azure_ai base for doc-intelligence models so api_base stays unset and both the rust bridge and the python get_complete_url fall back to the document-intelligence endpoint. This drops the earlier _rust_bridge_api_base reorder, which only covered the rust path and let the env silently override an explicit api_base * refactor(ocr): consolidate azure doc-intelligence detection; keep explicit api_base Extract is_azure_document_intelligence_model as the single source of truth for the azure_ai doc-intelligence sub-route so the check is no longer duplicated across _prepare_ocr_request and _rust_bridge_api_base, and gate the dynamic_api_base suppression on the caller not supplying an api_base so an explicit endpoint is always honoured. Restore xai to the realtime PROVIDERS as a documented disabled entry instead of dropping it silently, and add a regression test pinning doc-intelligence api_base resolution. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Mubashir Osmani Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/azure_ai/ocr/common_utils.py | 13 +- litellm/llms/deepseek/chat/transformation.py | 18 +- litellm/ocr/main.py | 15 +- tests/e2e/bob_the_builder.py | 247 ++++++++++++++++++ tests/e2e/conftest.py | 7 + tests/e2e/docker-compose.yml | 4 + tests/e2e/e2e_gateway.py | 21 ++ .../realtime/REALTIME_COVERAGE_MATRIX.md | 66 +++++ .../e2e/llm_translation/realtime/conftest.py | 37 +++ .../fixtures/weather_question_24k.wav | Bin .../realtime/pipecat_service.py | 0 .../realtime/realtime_client.py | 96 +++++-- .../realtime/test_realtime_e2e.py | 14 +- .../test_realtime_pipecat_audio_e2e.py | 23 +- .../realtime/test_realtime_pipecat_e2e.py | 8 +- .../test_deepseek_reasoning_e2e.py | 29 +- .../e2e/llm_translation/test_ocr_rust_e2e.py | 23 +- .../test_provider_features_e2e.py | 27 +- tests/e2e/models.py | 1 + .../e2e/realtime/REALTIME_COVERAGE_MATRIX.md | 55 ---- tests/e2e/realtime/conftest.py | 20 -- ...cr_azure_document_intelligence_api_base.py | 85 ++++++ 22 files changed, 636 insertions(+), 173 deletions(-) create mode 100644 tests/e2e/bob_the_builder.py create mode 100644 tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md create mode 100644 tests/e2e/llm_translation/realtime/conftest.py rename tests/e2e/{ => llm_translation}/realtime/fixtures/weather_question_24k.wav (100%) rename tests/e2e/{ => llm_translation}/realtime/pipecat_service.py (100%) rename tests/e2e/{ => llm_translation}/realtime/realtime_client.py (69%) rename tests/e2e/{ => llm_translation}/realtime/test_realtime_e2e.py (92%) rename tests/e2e/{ => llm_translation}/realtime/test_realtime_pipecat_audio_e2e.py (95%) rename tests/e2e/{ => llm_translation}/realtime/test_realtime_pipecat_e2e.py (97%) delete mode 100644 tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md delete mode 100644 tests/e2e/realtime/conftest.py create mode 100644 tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py diff --git a/litellm/llms/azure_ai/ocr/common_utils.py b/litellm/llms/azure_ai/ocr/common_utils.py index d1d5b80b78d..14b77338fd7 100644 --- a/litellm/llms/azure_ai/ocr/common_utils.py +++ b/litellm/llms/azure_ai/ocr/common_utils.py @@ -13,6 +13,17 @@ if TYPE_CHECKING: from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig +def is_azure_document_intelligence_model(model: str) -> bool: + """Whether an azure_ai OCR model routes to Azure Document Intelligence. + + Azure AI exposes two OCR services on the same provider; the sub-route in the + model name (`azure_ai/doc-intelligence/`) selects Document Intelligence + over Mistral OCR. This is the single source of truth for that routing decision. + """ + lowered = model.lower() + return "doc-intelligence" in lowered or "documentintelligence" in lowered + + def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]: """ Determine which Azure AI OCR configuration to use based on the model name. @@ -41,7 +52,7 @@ def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]: from litellm.llms.azure_ai.ocr.transformation import AzureAIOCRConfig # Check for Azure Document Intelligence models - if "doc-intelligence" in model or "documentintelligence" in model: + if is_azure_document_intelligence_model(model): verbose_logger.debug(f"Routing {model} to Azure Document Intelligence OCR config") return AzureDocumentIntelligenceOCRConfig() diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 7a548136f2a..525de1476e2 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -35,7 +35,9 @@ class DeepSeekChatConfig(OpenAIGPTConfig): Map OpenAI params to DeepSeek params. Handles `thinking` and `reasoning_effort` parameters for DeepSeek reasoner models. - DeepSeek only supports `{"type": "enabled"}` - no budget_tokens like Anthropic. + DeepSeek supports `{"type": "enabled"}` and `{"type": "disabled"}` - no budget_tokens + like Anthropic. `reasoning_effort="none"` is the OpenAI-style way to ask for thinking + off, so it maps to `{"type": "disabled"}`; any other effort keeps thinking on. Reference: https://api-docs.deepseek.com/guides/thinking_mode """ @@ -47,15 +49,13 @@ class DeepSeekChatConfig(OpenAIGPTConfig): thinking_value = optional_params.pop("thinking", None) reasoning_effort = optional_params.pop("reasoning_effort", None) - # Handle thinking parameter - only accept {"type": "enabled"} - if thinking_value is not None: - if isinstance(thinking_value, dict) and thinking_value.get("type") == "enabled": - # DeepSeek only accepts {"type": "enabled"}, ignore budget_tokens - optional_params["thinking"] = {"type": "enabled"} + # Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens + if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"): + optional_params["thinking"] = {"type": thinking_value["type"]} - # Handle reasoning_effort - map to thinking enabled - elif reasoning_effort is not None and reasoning_effort != "none": - optional_params["thinking"] = {"type": "enabled"} + # Otherwise fall back to reasoning_effort: "none" disables, anything else enables + elif reasoning_effort is not None: + optional_params["thinking"] = {"type": "disabled" if reasoning_effort == "none" else "enabled"} return optional_params diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5716155361d..38f3f804e10 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -17,6 +17,9 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.azure_ai.ocr.common_utils import ( + is_azure_document_intelligence_model, +) from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.rust_bridge import ocr as rust_ocr_bridge @@ -83,6 +86,8 @@ def _prepare_ocr_request( if doc_type not in ["document_url", "image_url"]: raise ValueError(f"Invalid document type: {doc_type}. Must be 'document_url', 'image_url', or 'file'") + caller_supplied_api_base = api_base is not None + ( model, custom_llm_provider, @@ -95,9 +100,14 @@ def _prepare_ocr_request( api_key=api_key, ) + suppress_dynamic_api_base = ( + not caller_supplied_api_base + and custom_llm_provider == "azure_ai" + and is_azure_document_intelligence_model(model) + ) if dynamic_api_key: api_key = dynamic_api_key - if dynamic_api_base: + if dynamic_api_base and not suppress_dynamic_api_base: api_base = dynamic_api_base ocr_provider_config = ProviderConfigManager.get_provider_ocr_config( @@ -191,8 +201,7 @@ def _rust_bridge_api_base( if prepared_request.api_base is not None: return prepared_request.api_base if prepared_request.custom_llm_provider == "azure_ai": - model = prepared_request.model.lower() - if "doc-intelligence" in model or "documentintelligence" in model: + if is_azure_document_intelligence_model(prepared_request.model): return resolve_secret("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") return resolve_secret("AZURE_AI_API_BASE") return None diff --git a/tests/e2e/bob_the_builder.py b/tests/e2e/bob_the_builder.py new file mode 100644 index 00000000000..18aff2edc98 --- /dev/null +++ b/tests/e2e/bob_the_builder.py @@ -0,0 +1,247 @@ +"""Bob the builder: on a red e2e run, ask Devin to fix the failing tests. + +Wired as a ``pytest_sessionfinish`` step (see ``conftest.py``). When the run went +red and remediation is enabled, it hands the failing tests plus their captured +tracebacks to Devin *through the LiteLLM proxy's own MCP gateway* -- the same +gateway + master key the suite already uses -- so Devin files a Linear ticket per +failure and opens fix PRs. Nothing new ships in the runner pod: the proxy already +registers the ``devin`` MCP server and holds ``DEVIN_API_KEY``, injecting it +upstream, so this process only needs the proxy key it always has. + +Opt-in via ``E2E_DEVIN_REMEDIATION=1`` so a normal local ``pytest tests/e2e`` run +never spawns a Devin session. ``DEVIN_DRY_RUN=1`` prints the prompt it would send +and makes no call. Everything is best-effort: any error here is logged and +swallowed so the run's exit status still reflects the tests, not remediation. +""" + +from __future__ import annotations + +import hashlib +import os +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol, cast + +import pytest +from pydantic import BaseModel, ConfigDict + +from e2e_config import MASTER_KEY, PROXY_BASE_URL +from e2e_http import Success +from transport import HttpTransport + +REMEDIATION_ENV = "E2E_DEVIN_REMEDIATION" +_LIST_PATH = "/mcp-rest/tools/list" +_CALL_PATH = "/mcp-rest/tools/call" + + +@dataclass(frozen=True, slots=True) +class Failure: + """One failed test: its pytest node id and the captured failure text.""" + + nodeid: str + detail: str + + +@dataclass(frozen=True, slots=True) +class Config: + server: str + create_tool: str + linear_team: str + target_repo: str + target_ref: str + max_failures: int + max_detail_chars: int + tags: tuple[str, ...] + dry_run: bool + + +class _NoParams(BaseModel): + pass + + +class _McpToolInfo(BaseModel): + model_config = ConfigDict(extra="allow") + server_name: str | None = None + alias: str | None = None + + +class _McpTool(BaseModel): + model_config = ConfigDict(extra="allow") + name: str + mcp_info: _McpToolInfo | None = None + + +class _McpToolsList(BaseModel): + model_config = ConfigDict(extra="allow") + tools: tuple[_McpTool, ...] = () + + +class _DevinSessionArgs(BaseModel): + prompt: str + title: str + tags: list[str] + + +class _ToolCallBody(BaseModel): + name: str + arguments: _DevinSessionArgs + + +class _ToolCallResult(BaseModel): + model_config = ConfigDict(extra="allow") + + +class _Report(Protocol): + @property + def nodeid(self) -> str: ... + + @property + def longreprtext(self) -> str: ... + + +class _TerminalReporter(Protocol): + stats: Mapping[str, Sequence[_Report]] + + +def _env(name: str, default: str) -> str: + value = os.environ.get(name, "").strip() + return value or default + + +def load_config() -> Config: + raw_tags = _env("DEVIN_TAGS", "e2e,stage") + return Config( + server=_env("DEVIN_MCP_SERVER", "devin"), + create_tool=_env("DEVIN_SESSION_TOOL", "devin_session_create"), + linear_team=_env("DEVIN_LINEAR_TEAM", "LIT"), + target_repo=_env("DEVIN_TARGET_REPO", "BerriAI/litellm"), + target_ref=_env("DEVIN_TARGET_REF", "litellm_internal_staging"), + max_failures=int(_env("DEVIN_MAX_FAILURES", "50")), + max_detail_chars=int(_env("DEVIN_MAX_DETAIL_CHARS", "3000")), + tags=tuple(t.strip() for t in raw_tags.split(",") if t.strip()), + dry_run=_env("DEVIN_DRY_RUN", "0") == "1", + ) + + +def collect_failures(session: pytest.Session, max_detail_chars: int) -> tuple[Failure, ...]: + """Pull the failed and errored tests (with their tracebacks) off the run's + terminal reporter. Returns empty when nothing failed or the reporter is + absent (e.g. a skipped, proxy-less session).""" + plugin: object = session.config.pluginmanager.getplugin("terminalreporter") + if plugin is None: + return () + reporter = cast(_TerminalReporter, plugin) + reports = (*reporter.stats.get("failed", ()), *reporter.stats.get("error", ())) + return tuple( + Failure(nodeid=r.nodeid, detail=r.longreprtext.strip()[-max_detail_chars:]) for r in reports + ) + + +def dedup_tag(failures: tuple[Failure, ...]) -> str: + """Stable short tag identifying this exact set of failing tests, so repeated + nightly runs on the same failures reference one body of work.""" + joined = "\n".join(sorted(f.nodeid for f in failures)) + return "e2e-fail-" + hashlib.sha256(joined.encode()).hexdigest()[:12] + + +def _revision() -> str: + for candidate in (Path(__file__).parent / ".litellm-revision", Path("/app/e2e/.litellm-revision")): + try: + return candidate.read_text(encoding="utf-8").strip() + except OSError: + continue + return _env("E2E_REVISION", "unknown") + + +def build_prompt(cfg: Config, failures: tuple[Failure, ...], tag: str) -> str: + shown = failures[: cfg.max_failures] + header = ( + f"The LiteLLM end-to-end suite failed on the " + f"{_env('E2E_ENVIRONMENT', 'stage')} proxy. Source repo {cfg.target_repo} " + f"at revision {_revision()} (branch {cfg.target_ref}). {len(failures)} " + f"test(s) failed" + + (f"; the first {len(shown)} are shown" if len(shown) < len(failures) else "") + + ".\n\n" + ) + task = ( + "For each failing test below:\n" + f"1. Open a Linear ticket under the {cfg.linear_team} team describing the " + "failure (test id, the assertion/error, likely cause), unless an open " + "ticket for that same test already exists -- do not create duplicates.\n" + f"2. Fix it in {cfg.target_repo}, branching off {cfg.target_ref} and " + "following the repo's CONTRIBUTING and CLAUDE.md conventions (meaningful " + "regression coverage, conventional commits, run the suite locally), then " + "open a PR that references the Linear ticket.\n" + "3. Prefer one focused PR per failing test; if several share a root cause, " + "group them and say so.\n" + f"Before starting, search existing sessions/PRs tagged '{tag}' or " + "referencing these test ids and continue that work instead of restarting.\n\n" + "Failing tests and their captured output:\n" + ) + blocks = [f"### {i}. {f.nodeid}\n```\n{f.detail}\n```\n" for i, f in enumerate(shown, start=1)] + return header + task + "\n".join(blocks) + + +def _resolve_tool_name(transport: HttpTransport, cfg: Config) -> str | None: + """Find Devin's create-session tool on the gateway. The proxy prefixes tools + with the server alias, so match by suffix and (when present) the owning + server.""" + result = transport.get( + _LIST_PATH, headers=transport.master, params=_NoParams(), response_type=_McpToolsList + ) + if not isinstance(result, Success): + print(f"bob_the_builder: could not list gateway MCP tools: {result}") + return None + for tool in result.data.tools: + owner = tool.mcp_info.server_name or tool.mcp_info.alias if tool.mcp_info else None + if (owner is None or owner == cfg.server) and ( + tool.name == cfg.create_tool or tool.name.endswith(cfg.create_tool) + ): + return tool.name + print( + f"bob_the_builder: no '{cfg.create_tool}' tool for server '{cfg.server}' on the gateway; " + f"saw {[t.name for t in result.data.tools]}" + ) + return None + + +def remediate(session: pytest.Session) -> None: + """Entry point called from ``pytest_sessionfinish``. No-op unless remediation + is enabled and the run actually had failures.""" + if os.environ.get(REMEDIATION_ENV) != "1": + return + cfg = load_config() + failures = collect_failures(session, cfg.max_detail_chars) + if not failures: + return + + tag = dedup_tag(failures) + title = f"Fix {len(failures)} failing LiteLLM e2e test(s) [{tag}]" + prompt = build_prompt(cfg, failures, tag) + args = _DevinSessionArgs(prompt=prompt, title=title, tags=[*cfg.tags, tag]) + + if cfg.dry_run: + print("bob_the_builder: DRY RUN -- would create a Devin session:") + print(f" server : {cfg.server}\n tool : {cfg.create_tool}\n title : {title}") + print(f" tags : {args.tags}\n---- prompt ----\n{prompt}") + return + + try: + transport = HttpTransport(base_url=PROXY_BASE_URL, master_key=MASTER_KEY) + tool_name = _resolve_tool_name(transport, cfg) + if tool_name is None: + return + result = transport.post( + _CALL_PATH, + headers=transport.master, + json=_ToolCallBody(name=tool_name, arguments=args), + response_type=_ToolCallResult, + ) + if isinstance(result, Success): + print(f"bob_the_builder: created Devin session for {len(failures)} failure(s) [{tag}]") + print(result.data.model_dump_json()) + else: + print(f"bob_the_builder: Devin session call failed: {result}") + except Exception as exc: # noqa: BLE001 - remediation must never fail the run + print(f"bob_the_builder: remediation error (ignored): {exc}") diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 9ca5840df24..82f2604d492 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -107,6 +107,13 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if spend_dir in sys.path: sys.path.remove(spend_dir) + try: + from bob_the_builder import remediate + + remediate(session) + except Exception as exc: # noqa: BLE001 - remediation is best-effort + print(f"devin remediation skipped: {exc}") + @pytest.fixture def resources(client: GatewayProvider) -> Iterator[ResourceManager]: diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index cdf5d6cbf6f..195badc5285 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -25,6 +25,10 @@ configs: fallbacks: - gemini-2.5-flash: ["gpt-5.5", "claude-haiku-4-5"] + finetune_settings: + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + model_list: - model_name: gpt-5.5 litellm_params: diff --git a/tests/e2e/e2e_gateway.py b/tests/e2e/e2e_gateway.py index d62c9c4b17b..05f83ecc085 100644 --- a/tests/e2e/e2e_gateway.py +++ b/tests/e2e/e2e_gateway.py @@ -28,6 +28,9 @@ from models import ( CustomerDeleteBody, EmbedBody, EmbedResponse, + FileListResponse, + FineTuningJobsParams, + FineTuningJobsResponse, KeyDeleteBody, KeyGenerateBody, KeyGenerateResponse, @@ -120,6 +123,24 @@ class Gateway: ) ).data + def list_files(self, key: str) -> Result[FileListResponse]: + return self.transport.get( + "/v1/files", + headers=self.transport.bearer(key), + params=NoBody(), + response_type=FileListResponse, + ) + + def list_fine_tuning_jobs( + self, key: str, params: FineTuningJobsParams + ) -> Result[FineTuningJobsResponse]: + return self.transport.get( + "/v1/fine_tuning/jobs", + headers=self.transport.bearer(key), + params=params, + response_type=FineTuningJobsResponse, + ) + def create_model( self, model_name: str, diff --git a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md new file mode 100644 index 00000000000..ff8b3441d86 --- /dev/null +++ b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md @@ -0,0 +1,66 @@ +# Realtime e2e coverage + +Live tests for the proxy realtime websocket endpoint (`/v1/realtime`). One +GA-speaking websocket client drives every provider; the proxy normalizes each +provider's stream into the OpenAI GA event schema, so the same assertions hold +across providers and only the model alias changes. + +## What is asserted + +For each configured provider, `test_text_conversation` checks the session +lifecycle (`session.created`, then `session.update` echoed by `session.updated`), +the canonical response sequence (`response.created`, `response.output_item.added`, +through `response.done`), that the streamed deltas reconstruct a non-empty +transcript, and that `response.done` carries normalized usage. + +`test_tool_call_round_trip` checks the full tool path: the model emits a +normalized `response.function_call_arguments.done` with valid JSON arguments and +a matching `function_call` output item, the test sends a `function_call_output` +back, and the follow-up response incorporates the result (the temperature 72 +appears). + +`test_realtime_pipecat_e2e` is a realism layer that drives the same providers +through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy) +rather than speaking the protocol by hand. Its assertions are coarse (the tool +callback fired, assistant text was produced); the raw-websocket suite is the +source of truth. It skips unless `pipecat-ai` is installed +(`uv pip install "pipecat-ai[openai]"`). + +## Provisioning + +The suite registers every provider's realtime deployment through `/model/new` at +session start (the `realtime_models` fixture) and deletes them on teardown, so it +never depends on a static or misconfigured gateway `model_list`. Each deployment +is created with `model_info.mode: realtime` and marker-unique names, and its +`litellm_params` point the credentials at `os.environ/*` refs the gateway resolves +at call time. The provider table below is the source of truth; edit `PROVIDERS` in +`realtime_client.py` to change a model or add one. + +| provider | model alias | upstream model | +|----------|-------------|----------------| +| openai | `openai-realtime` | `openai/gpt-realtime-2` | +| azure | `azure-realtime` | `azure/gpt-realtime-2` (GA protocol) | +| gemini | `gemini-realtime` | `gemini/gemini-3.1-flash-live-preview` | +| vertex_ai | `vertex-realtime` | `vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025` | + +Bedrock and xai (`xai/grok-4-1-fast-non-reasoning`) are supported by the proxy but +kept commented out in `PROVIDERS` until they pass end-to-end here; re-enable them by +uncommenting their entry. + +Every provider is provisioned and asserted; the suite never skips a provider. Per +`tests/e2e/CLAUDE.md` the only sanctioned skip is the whole-suite proxy-liveness +skip, so a provider whose credentials or upstream realtime model are missing on the +gateway is a hard failure, not a skip. Give the gateway each provider's credentials +to turn its tests green. + +## Running + +Start a proxy with the provider keys set in its environment (the suite registers +the deployments itself), then + +``` +uv run pytest tests/e2e/llm_translation/realtime/ -v +``` + +The whole suite skips only when no proxy answers `GET /health/liveliness` at +`LITELLM_PROXY_URL` (default `http://localhost:4000`). diff --git a/tests/e2e/llm_translation/realtime/conftest.py b/tests/e2e/llm_translation/realtime/conftest.py new file mode 100644 index 00000000000..15cd789664e --- /dev/null +++ b/tests/e2e/llm_translation/realtime/conftest.py @@ -0,0 +1,37 @@ +"""Realtime suite's `client` and `realtime_models` fixtures. + +The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker +live in the parent tests/e2e/conftest.py. RealtimeClient holds the shared Gateway, +so the `resources` fixture cleans up keys this suite creates. + +`realtime_models` registers every provider's realtime deployment through /model/new +at session start and deletes them at teardown, so the suite provisions the models it +uses through the management endpoints instead of depending on a static (or +misconfigured) gateway model_list. +""" + +from collections.abc import Iterator + +import pytest + +from realtime_client import PROVIDERS, RealtimeClient, build_client + + +@pytest.fixture(scope="session") +def client() -> RealtimeClient: + return build_client() + + +@pytest.fixture(scope="session") +def realtime_models(client: RealtimeClient) -> Iterator[dict[str, str]]: + """Provision each provider's realtime deployment via /model/new and yield a + provider-id -> model-name map the tests connect with; delete them on teardown. + Every provider is provisioned (never skipped): a provider whose credentials or + upstream model are missing on the gateway hard-fails its test, per the suite's + fail-on-behavior contract in tests/e2e/CLAUDE.md.""" + records = tuple((provider.id, *client.provision(provider)) for provider in PROVIDERS) + try: + yield {provider_id: model_name for provider_id, model_name, _ in records} + finally: + for _, _, model_id in records: + client.gateway.delete_model(model_id) diff --git a/tests/e2e/realtime/fixtures/weather_question_24k.wav b/tests/e2e/llm_translation/realtime/fixtures/weather_question_24k.wav similarity index 100% rename from tests/e2e/realtime/fixtures/weather_question_24k.wav rename to tests/e2e/llm_translation/realtime/fixtures/weather_question_24k.wav diff --git a/tests/e2e/realtime/pipecat_service.py b/tests/e2e/llm_translation/realtime/pipecat_service.py similarity index 100% rename from tests/e2e/realtime/pipecat_service.py rename to tests/e2e/llm_translation/realtime/pipecat_service.py diff --git a/tests/e2e/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py similarity index 69% rename from tests/e2e/realtime/realtime_client.py rename to tests/e2e/llm_translation/realtime/realtime_client.py index 07dfd76108b..ef7834d6bbe 100644 --- a/tests/e2e/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -11,19 +11,19 @@ models, matching the suite's no-raw-dicts rule. from __future__ import annotations import time -from collections.abc import Generator +from collections.abc import Generator, Mapping from contextlib import contextmanager from dataclasses import dataclass from typing import Any, TypeVar from urllib.parse import urlencode -import pytest from pydantic import BaseModel, ConfigDict from websockets.sync.client import connect from websockets.sync.connection import Connection -from e2e_config import PROXY_BASE_URL +from e2e_config import PROXY_BASE_URL, unique_marker from e2e_gateway import Gateway, build_gateway +from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) @@ -41,25 +41,76 @@ def realtime_ws_url(model: str) -> str: @dataclass(frozen=True, slots=True) class RealtimeProvider: + """A realtime provider the suite exercises. `litellm_params` is the deployment + the suite registers through /model/new (the gateway resolves the os.environ/* + credential refs), so the suite is self-contained and never depends on a static + gateway model_list. Every provider here is provisioned and asserted: per + tests/e2e/CLAUDE.md the suite never skips a provider, so a provider whose + credentials or upstream realtime model are missing on the gateway is a hard + failure, not a skip.""" + id: str - model: str + alias: str + litellm_params: LiteLLMParamsBody PROVIDERS = ( - RealtimeProvider("openai", "openai-realtime"), - RealtimeProvider("azure", "azure-realtime"), - RealtimeProvider("gemini", "gemini-realtime"), - RealtimeProvider("vertex_ai", "vertex-realtime"), - # RealtimeProvider("bedrock", "bedrock-realtime"), # TODO: Enable this when Bedrock is passing - RealtimeProvider("xai", "xai-realtime"), + RealtimeProvider( + "openai", + "openai-realtime", + LiteLLMParamsBody( + model="openai/gpt-realtime-2", + api_key="os.environ/OPENAI_API_KEY", + ), + ), + RealtimeProvider( + "azure", + "azure-realtime", + LiteLLMParamsBody( + model="azure/gpt-realtime", + api_key="os.environ/AZURE_API_KEY", + api_version="2025-08-28", + realtime_protocol="GA", + ), + ), + RealtimeProvider( + "gemini", + "gemini-realtime", + LiteLLMParamsBody( + model="gemini/gemini-3.1-flash-live-preview", + api_key="os.environ/GEMINI_API_KEY", + ), + ), + RealtimeProvider( + "vertex_ai", + "vertex-realtime", + LiteLLMParamsBody( + model="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025", + vertex_location="us-central1", + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + ), + ), + # RealtimeProvider("bedrock", "bedrock-realtime", ...) # TODO: Enable when Bedrock is passing + # RealtimeProvider( + # "xai", + # "xai-realtime", + # LiteLLMParamsBody( + # model="xai/grok-4-1-fast-non-reasoning", + # api_key="os.environ/XAI_API_KEY", + # ), + # ), # TODO: Enable once xai Grok Voice realtime is passing end-to-end here ) -def skip_if_unconfigured( - provider: RealtimeProvider, configured: frozenset[str] -) -> None: - if provider.model not in configured: - pytest.skip(f"{provider.model} not configured on proxy") +def realtime_model(provider: RealtimeProvider, provisioned: Mapping[str, str]) -> str: + """Return the provisioned deployment name for this provider. Every provider in + PROVIDERS is provisioned at session start, so a missing entry is a harness bug, + never an environment skip - the suite hard-fails instead (see tests/e2e/CLAUDE.md).""" + model = provisioned.get(provider.id) + assert model is not None, ( + f"{provider.id} was not provisioned; the realtime_models fixture is broken" + ) + return model # ---- sent events ------------------------------------------------------- @@ -280,12 +331,17 @@ class RealtimeSession: class RealtimeClient: gateway: Gateway - def configured_models(self) -> frozenset[str]: - return frozenset( - entry.model_name - for entry in self.gateway.model_info() - if entry.model_info.mode == "realtime" + def provision(self, provider: RealtimeProvider) -> tuple[str, str]: + """Register this provider's realtime deployment through /model/new and return + (model_name, model_id). The name is marker-unique so it never collides with a + same-named deployment already on the shared proxy, and mode=realtime makes it + show up as a realtime model on /model/info. add_deployment runs synchronously, + so the deployment is connectable as soon as this returns.""" + model_name = f"{provider.alias}-{unique_marker()}" + model_id = self.gateway.create_model( + model_name, provider.litellm_params, mode="realtime" ) + return model_name, model_id @contextmanager def connect( diff --git a/tests/e2e/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py similarity index 92% rename from tests/e2e/realtime/test_realtime_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_e2e.py index 01356900141..6aaffdd208e 100644 --- a/tests/e2e/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -31,7 +31,7 @@ from realtime_client import ( SessionUpdate, function_call_item, parse_last, - skip_if_unconfigured, + realtime_model, transcript, user_message, ) @@ -62,12 +62,12 @@ class WeatherResult(BaseModel): def test_text_conversation( client: RealtimeClient, scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - with client.connect(key=scoped_key, model=provider.model) as session: + with client.connect(key=scoped_key, model=model) as session: created = session.collect_until("session.created", timeout=20) assert created[-1].type == "session.created" @@ -99,12 +99,12 @@ def test_text_conversation( def test_tool_call_round_trip( client: RealtimeClient, scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - with client.connect(key=scoped_key, model=provider.model) as session: + with client.connect(key=scoped_key, model=model) as session: session.collect_until("session.created", timeout=20) session.send( SessionUpdate( diff --git a/tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py similarity index 95% rename from tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py index d9d7c744f66..31c038b4e02 100644 --- a/tests/e2e/realtime/test_realtime_pipecat_audio_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e.py @@ -31,7 +31,7 @@ from realtime_client import ( PROVIDERS, RealtimeProvider, _ws_base_url, - skip_if_unconfigured, + realtime_model, ) pytestmark = pytest.mark.e2e @@ -189,13 +189,13 @@ async def _run_pipeline( @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_server_vad( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Session is configured with server-VAD; bot must respond to a text prompt.""" - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, provider.model)) + tool_called, got_text, _ = asyncio.run(_run_pipeline(scoped_key, model)) assert tool_called, "get_weather tool was not invoked" assert got_text, "no assistant text frames produced" @@ -204,16 +204,16 @@ def test_pipecat_server_vad( @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_audio_output( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Bot must produce at least one non-empty TTS audio frame.""" - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) _, got_text, audio_bytes = asyncio.run( _run_pipeline( scoped_key, - provider.model, + model, prompt="Say hello in one short sentence.", timeout=30.0, ) @@ -328,7 +328,7 @@ async def _run_audio_input_pipeline( @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_server_vad_audio_input( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: """Stream a real PCM16 WAV fixture; server VAD must detect speech end and respond. @@ -337,12 +337,11 @@ def test_pipecat_server_vad_audio_input( → server-VAD turn detection → response.create (auto) → assistant reply. No LLMRunFrame is sent — the response must be triggered entirely by VAD. """ - if not WEATHER_WAV.exists(): - pytest.skip(f"audio fixture not found: {WEATHER_WAV}") - skip_if_unconfigured(provider, configured_models) + assert WEATHER_WAV.exists(), f"audio fixture not found: {WEATHER_WAV}" + model = realtime_model(provider, realtime_models) got_text, audio_bytes = asyncio.run( - _run_audio_input_pipeline(scoped_key, provider.model) + _run_audio_input_pipeline(scoped_key, model) ) assert got_text, "server VAD did not trigger a response (no assistant text)" diff --git a/tests/e2e/realtime/test_realtime_pipecat_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py similarity index 97% rename from tests/e2e/realtime/test_realtime_pipecat_e2e.py rename to tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py index 1068c54fdec..799958ef4e3 100644 --- a/tests/e2e/realtime/test_realtime_pipecat_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_pipecat_e2e.py @@ -29,7 +29,7 @@ from realtime_client import ( PROVIDERS, RealtimeProvider, _ws_base_url, - skip_if_unconfigured, + realtime_model, ) pytestmark = pytest.mark.e2e @@ -123,12 +123,12 @@ async def _run_pipeline(key: str, model: str) -> tuple[bool, bool]: @pytest.mark.parametrize("provider", PROVIDER_PARAMS) def test_pipecat_tool_smoke( scoped_key: str, - configured_models: frozenset[str], + realtime_models: dict[str, str], provider: RealtimeProvider, ) -> None: - skip_if_unconfigured(provider, configured_models) + model = realtime_model(provider, realtime_models) - tool_called, produced_text = asyncio.run(_run_pipeline(scoped_key, provider.model)) + tool_called, produced_text = asyncio.run(_run_pipeline(scoped_key, model)) assert tool_called, "pipecat did not invoke the get_weather callback" assert produced_text, "pipecat produced no assistant text frames" diff --git a/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py b/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py index f8f229aa2a7..5adb8c24f9f 100644 --- a/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py +++ b/tests/e2e/llm_translation/test_deepseek_reasoning_e2e.py @@ -2,17 +2,14 @@ DeepSeek's reasoner defaults thinking ON and surfaces the chain as ``message.reasoning_content``. Two documented ways to disable it are -``reasoning_effort="none"`` and ``thinking={"type": "disabled"}``. Today the -DeepSeek param mapper (``litellm/llms/deepseek/chat/transformation.py`` -``map_openai_params``) drops both without forwarding any disable signal, so the -outbound body carries no ``thinking`` key and DeepSeek keeps thinking on; the -response still comes back with ``reasoning_content``. That is the product gap -tracked by LIT-3686 / GH #27453. +``reasoning_effort="none"`` and ``thinking={"type": "disabled"}``. The DeepSeek +param mapper (``litellm/llms/deepseek/chat/transformation.py`` +``map_openai_params``) forwards both as ``thinking={"type": "disabled"}`` so the +outbound body carries a real disable signal and ``deepseek-reasoner`` returns no +``reasoning_content``. This is the behavior tracked by LIT-3686 / GH #27453. The control case proves the model and path work (reasoning is returned when -nothing asks to disable it), so the two disable assertions are meaningful. Those -two are marked xfail(strict) until the mapper forwards a real disable signal; an -xpass then alerts that the fix landed. +nothing asks to disable it), so the two disable assertions are meaningful. Requires DEEPSEEK_API_KEY on the proxy (tests/e2e/.env). No skip gate: once the proxy is up, a failure here is real, per the suite's hard-fail contract. @@ -74,13 +71,6 @@ class TestDeepSeekReasoningDisable: f"disable param, so the disable assertions below can't be trusted: {response}" ) - @pytest.mark.xfail( - strict=True, - reason=( - "LIT-3686 / GH #27453: DeepSeek reasoning_effort='none' and " - "thinking type='disabled' are silently dropped; reasoning not disabled" - ), - ) def test_reasoning_effort_none_disables_reasoning( self, client: PassthroughClient, resources: ResourceManager ) -> None: @@ -103,13 +93,6 @@ class TestDeepSeekReasoningDisable: f"is still present: {response}" ) - @pytest.mark.xfail( - strict=True, - reason=( - "LIT-3686 / GH #27453: DeepSeek reasoning_effort='none' and " - "thinking type='disabled' are silently dropped; reasoning not disabled" - ), - ) def test_thinking_disabled_disables_reasoning( self, client: PassthroughClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index 361bb5126a7..921010e5eae 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -86,16 +86,18 @@ class AzureDocIntelligenceOcr: @dataclass(frozen=True, slots=True) class VertexOcr: + """Vertex AI OCR (Mistral publisher). Only the location (not a secret) is set; + the project and credentials are left unset so the gateway resolves VERTEXAI_PROJECT + and VERTEXAI_CREDENTIALS from its own environment by name, keeping every secret on + the gateway like the azure_ai cases above. This is deliberate: the OCR path reads + vertex_project verbatim from litellm_params and never unwraps an `os.environ/*` + ref, so passing one would put the literal string in the request URL.""" + model: str location: str def litellm_params(self) -> LiteLLMParamsBody: - return LiteLLMParamsBody( - model=self.model, - vertex_project="os.environ/VERTEXAI_PROJECT", - vertex_location=self.location, - vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", - ) + return LiteLLMParamsBody(model=self.model, vertex_location=self.location) @dataclass(frozen=True, slots=True) @@ -113,7 +115,7 @@ RUST_OCR_CASES: tuple[_OcrCase, ...] = ( ), _OcrCase( "azure-ai", - AzureAiOcr("azure_ai/mistral-document-ai-2505"), + AzureAiOcr("azure_ai/mistral-document-ai-2512"), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), _OcrCase( @@ -126,11 +128,6 @@ RUST_OCR_CASES: tuple[_OcrCase, ...] = ( VertexOcr("vertex_ai/mistral-ocr-2505", "us-central1"), OcrDocument(type="document_url", document_url=TEST_PDF_URL), ), - _OcrCase( - "vertex-deepseek", - VertexOcr("vertex_ai/deepseek-ocr-maas", "global"), - OcrDocument(type="image_url", image_url=TEST_IMAGE_URL), - ), ) _CASE_IDS = tuple(case.suffix for case in RUST_OCR_CASES) @@ -155,3 +152,5 @@ class TestRustOcrGateway: response = unwrap(endpoints_client.gateway.ocr(key, OcrBody(model=model, document=case.document))) _assert_ocr_document(response) + + diff --git a/tests/e2e/llm_translation/test_provider_features_e2e.py b/tests/e2e/llm_translation/test_provider_features_e2e.py index cf05a4306b4..d272fffa9b8 100644 --- a/tests/e2e/llm_translation/test_provider_features_e2e.py +++ b/tests/e2e/llm_translation/test_provider_features_e2e.py @@ -3,11 +3,13 @@ Each case asserts the feature took effect, not just a 200. service_tier is an OpenAI concept. The proxy forwards it and the provider echoes -the tier back on the response, so sending a non-default tier ("flex") and reading -it back off ``service_tier`` proves the param was honored end to end; litellm's own -default injection would report "default", so a "flex" echo can only come from the -request being forwarded. Bedrock and Vertex do not accept service_tier, so that -cell is OpenAI-only by design. +the tier back on the response, so sending a non-default tier ("priority") and +reading it back off ``service_tier`` proves the param was honored end to end; +litellm's own default injection (and service_tier="auto") both report "default", +so a "priority" echo can only come from the request being forwarded. "flex" is +avoided here because it is capacity-constrained and returns a transient 429 when +flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so +that cell is OpenAI-only by design. Prompt caching is asserted through provider prompt-cache usage tokens. The deterministic path is explicit ``cache_control`` on an Anthropic-family model @@ -22,7 +24,7 @@ scope here and covered only by the explicit-cache-control Bedrock case. from __future__ import annotations import pytest -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field from e2e_config import unique_marker from e2e_http import unwrap @@ -32,7 +34,7 @@ from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -SERVICE_TIER = "flex" +SERVICE_TIER = "priority" CACHE_MIN_READ_TOKENS = 1 @@ -51,10 +53,21 @@ class RichMessage(BaseModel): content: list[CacheTextBlock] +class CacheDirective(BaseModel): + """litellm per-request cache control. ``no-cache`` forces the proxy to skip its + own response cache and make a fresh provider call, so the second identical + request actually reaches Bedrock and reads the provider prompt cache instead of + being served the first response verbatim (which would report cache_read=0).""" + + model_config = ConfigDict(populate_by_name=True) + no_cache: bool = Field(default=True, alias="no-cache") + + class CacheChatBody(BaseModel): model: str messages: list[RichMessage] max_tokens: int + cache: CacheDirective = CacheDirective() def cacheable_prefix() -> str: diff --git a/tests/e2e/models.py b/tests/e2e/models.py index f287058b313..38778034de9 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -376,6 +376,7 @@ class LiteLLMParamsBody(BaseModel): api_key: str | None = None api_base: str | None = None api_version: str | None = None + realtime_protocol: str | None = None aws_region_name: str | None = None vertex_project: str | None = None vertex_location: str | None = None diff --git a/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md deleted file mode 100644 index 8624475d0de..00000000000 --- a/tests/e2e/realtime/REALTIME_COVERAGE_MATRIX.md +++ /dev/null @@ -1,55 +0,0 @@ -# Realtime e2e coverage - -Live tests for the proxy realtime websocket endpoint (`/v1/realtime`). One -GA-speaking websocket client drives every provider; the proxy normalizes each -provider's stream into the OpenAI GA event schema, so the same assertions hold -across providers and only the model alias changes. - -## What is asserted - -For each configured provider, `test_text_conversation` checks the session -lifecycle (`session.created`, then `session.update` echoed by `session.updated`), -the canonical response sequence (`response.created`, `response.output_item.added`, -through `response.done`), that the streamed deltas reconstruct a non-empty -transcript, and that `response.done` carries normalized usage. - -`test_tool_call_round_trip` checks the full tool path: the model emits a -normalized `response.function_call_arguments.done` with valid JSON arguments and -a matching `function_call` output item, the test sends a `function_call_output` -back, and the follow-up response incorporates the result (the temperature 72 -appears). - -`test_realtime_pipecat_e2e` is a realism layer that drives the same providers -through pipecat's GA `OpenAIRealtimeLLMService` (base_url pointed at the proxy) -rather than speaking the protocol by hand. Its assertions are coarse (the tool -callback fired, assistant text was produced); the raw-websocket suite is the -source of truth. It skips unless `pipecat-ai` is installed -(`uv pip install "pipecat-ai[openai]"`). - -## Provider status - -| provider | model alias | status | -|----------|-------------|--------| -| openai | `openai-realtime` | covered (in gateway config) | -| gemini | `gemini-realtime` | covered (in gateway config; needs Gemini Live API access) | -| azure | `azure-realtime` | gap: add to gateway config + AZURE creds | -| vertex_ai | `vertex-realtime` | gap: add to gateway config + Vertex creds | -| bedrock | `bedrock-realtime` | gap: add to gateway config + AWS creds | -| xai | `xai-realtime` | gap: add to gateway config + XAI_API_KEY | - -A provider whose alias is not present in the proxy's `/model/info` skips (skip on -environment). To enable one, add a `model_info.mode: realtime` entry under that -alias to `tests/e2e/gateway/litellm-config.yml` and give the proxy the -provider's credentials; the test then runs with no code change. - -## Running - -Start a proxy with the gateway config and the provider keys set in its -environment, then - -``` -uv run pytest tests/e2e/realtime/ -v -``` - -Tests skip when no proxy answers `GET /health/liveliness` at `LITELLM_PROXY_URL` -(default `http://localhost:4000`). diff --git a/tests/e2e/realtime/conftest.py b/tests/e2e/realtime/conftest.py deleted file mode 100644 index 4a5c4837a1a..00000000000 --- a/tests/e2e/realtime/conftest.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Realtime suite's `client` fixture. - -The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker -live in the parent tests/e2e/conftest.py. RealtimeClient holds the shared -Gateway, so the `resources` fixture cleans up keys this suite creates. -""" - -import pytest - -from realtime_client import RealtimeClient, build_client - - -@pytest.fixture(scope="session") -def client() -> RealtimeClient: - return build_client() - - -@pytest.fixture(scope="session") -def configured_models(client: RealtimeClient) -> frozenset[str]: - return client.configured_models() diff --git a/tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py b/tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py new file mode 100644 index 00000000000..0c8b1cc2836 --- /dev/null +++ b/tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py @@ -0,0 +1,85 @@ +""" +Regression tests for Azure Document Intelligence api_base resolution in OCR. + +`azure_ai` exposes two OCR services on one provider; the `doc-intelligence` +sub-route must resolve to `AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT`, not to the +generic `AZURE_AI_API_BASE` fallback that `get_llm_provider` injects. These tests +pin that routing and guard the backwards-compatibility contract that an explicitly +supplied api_base is always honoured. +""" + +from litellm.llms.azure_ai.ocr.common_utils import ( + is_azure_document_intelligence_model, +) +from litellm.ocr.main import _prepare_ocr_request, _rust_bridge_api_base + +_DOC = {"type": "document_url", "document_url": "https://example.com/doc.pdf"} +_DOC_INTELLIGENCE_ENDPOINT = "https://di.cognitiveservices.azure.com" +_AZURE_AI_API_BASE = "https://generic-azure-ai.example.com" + + +class _FakeLogging: + def update_from_kwargs(self, **kwargs: object) -> None: + return None + + +def _resolve_secret(name: str) -> str | None: + return { + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": _DOC_INTELLIGENCE_ENDPOINT, + "AZURE_AI_API_BASE": _AZURE_AI_API_BASE, + }.get(name) + + +def _prepare(model: str, api_base: str | None): + return _prepare_ocr_request( + model=model, + document=dict(_DOC), + api_key="test-key", + api_base=api_base, + timeout=None, + custom_llm_provider=None, + extra_headers=None, + kwargs={"litellm_logging_obj": _FakeLogging()}, + ) + + +class TestIsAzureDocumentIntelligenceModel: + def test_matches_doc_intelligence_route(self): + assert is_azure_document_intelligence_model("doc-intelligence/prebuilt-layout") + + def test_matches_documentintelligence_and_is_case_insensitive(self): + assert is_azure_document_intelligence_model("azure_ai/DocumentIntelligence/x") + + def test_does_not_match_mistral_route(self): + assert not is_azure_document_intelligence_model("mistral-document-ai-2505") + + +class TestDocIntelligenceApiBaseResolution: + def test_generic_azure_ai_base_does_not_hijack_doc_intelligence(self, monkeypatch): + """Without an explicit api_base, the AZURE_AI_API_BASE fallback must not + overwrite the endpoint, so it resolves to the Document Intelligence one.""" + monkeypatch.setenv("AZURE_AI_API_BASE", _AZURE_AI_API_BASE) + monkeypatch.delenv("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT", raising=False) + + prepared = _prepare("azure_ai/doc-intelligence/prebuilt-layout", None) + + assert prepared.api_base is None + assert _rust_bridge_api_base(prepared, _resolve_secret) == _DOC_INTELLIGENCE_ENDPOINT + + def test_explicit_api_base_is_honoured_for_doc_intelligence(self, monkeypatch): + """A caller-supplied api_base must always win, even for doc-intelligence.""" + monkeypatch.setenv("AZURE_AI_API_BASE", _AZURE_AI_API_BASE) + + custom = "https://my-di.cognitiveservices.azure.com" + prepared = _prepare("azure_ai/doc-intelligence/prebuilt-layout", custom) + + assert prepared.api_base == custom + assert _rust_bridge_api_base(prepared, _resolve_secret) == custom + + def test_generic_azure_ai_base_still_applies_to_mistral_ocr(self, monkeypatch): + """Non doc-intelligence azure_ai models keep using AZURE_AI_API_BASE.""" + monkeypatch.setenv("AZURE_AI_API_BASE", _AZURE_AI_API_BASE) + + prepared = _prepare("azure_ai/mistral-document-ai-2505", None) + + assert prepared.api_base == _AZURE_AI_API_BASE From 41e9cc491ed08a70b96ba1f77efb39de76680943 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:31:27 -0700 Subject: [PATCH 073/101] fix(bedrock-invoke): retain clear_tool_uses_20250919 context_management edits and emit context-management-2025-06-27 beta (LIT-3393) (#32658) * fix(bedrock-invoke): retain clear_tool_uses_20250919 context_management edits and emit context-management-2025-06-27 beta (LIT-3393) Copy of #29206 by oss-agent-shin, rebased onto litellm_internal_staging so CircleCI can run. Bedrock InvokeModel supports automatic tool-call clearing (clear_tool_uses_20250919) under the context-management-2025-06-27 beta, but LiteLLM stripped the edit and dropped the beta header, causing a Bedrock 400. This maps bedrock.context-management-2025-06-27 to itself in anthropic_beta_headers_config.json (bedrock_converse stays null) and rewrites _filter_context_management_for_bedrock_invoke around an allowlist of supported edit types that keeps each supported edit and adds its matching beta. * test(bedrock-invoke): restore beta-headers config cache with a shared fixture in LIT-3393 tests Greptile flagged that three of the four new tests reloaded the module-level beta-headers config into local mode without restoring it on teardown, leaking state into later tests in the same process. Move setup/teardown into a local_beta_headers_config fixture used by all four tests. --------- Co-authored-by: oss-agent-shin --- litellm/anthropic_beta_headers_config.json | 2 +- .../anthropic_claude3_transformation.py | 57 ++++-- .../test_anthropic_claude3_transformation.py | 171 ++++++++++++++++++ 3 files changed, 211 insertions(+), 19 deletions(-) diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 11fdb26e42d..3f6817f6e35 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -102,7 +102,7 @@ "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", - "context-management-2025-06-27": null, + "context-management-2025-06-27": "context-management-2025-06-27", "effort-2025-11-24": "effort-2025-11-24", "fast-mode-2026-02-01": null, "files-api-2025-04-14": null, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index f5309d521a9..daee3369a3c 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -489,24 +489,43 @@ class AmazonAnthropicClaudeMessagesConfig( if self._supports_tool_search_on_bedrock(model): beta_set.add("tool-search-tool-2025-10-19") + # Bedrock-InvokeModel-supported ``context_management.edits`` types and the + # ``anthropic-beta`` header that each one requires. ``clear_thinking_20251015`` + # is intentionally absent — it is LiteLLM-internal, consumed via + # ``_ensure_thinking_for_clear_thinking_context_management``, and forwarding + # the raw edit trips Bedrock's + # ``"context_management: Extra inputs are not permitted"`` 400. + # + # Bedrock InvokeModel DOES support ``clear_tool_uses_20250919`` under the + # ``context-management-2025-06-27`` beta. AWS docs: + # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md + _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Dict[str, str] = { + "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + "clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + } + @staticmethod def _filter_context_management_for_bedrock_invoke( anthropic_messages_request: Dict, beta_set: set, ) -> None: """ - Bedrock InvokeModel accepts ``context_management`` only when it carries - ``compact_20260112`` edits paired with the ``compact-2026-01-12`` - anthropic-beta header. Other edit types (notably ``clear_thinking_20251015``, - which Claude Code sends on every request) are LiteLLM-internal and would - cause Bedrock to 400 with ``"context_management: Extra inputs are not - permitted"``. + Filter ``context_management.edits`` to the subset that Bedrock InvokeModel + accepts and add the matching ``anthropic-beta`` header for each surviving + edit type. - Filter the edits list to the supported subset, add the beta header when - compact edits remain, and drop ``context_management`` entirely when no - supported edits are left so the safety-net allowlist can pass it through. + - ``compact_20260112`` -> ``compact-2026-01-12`` + - ``clear_tool_uses_20250919`` -> ``context-management-2025-06-27`` - Ref: https://github.com/BerriAI/litellm/issues/27532 + Other edit types (notably ``clear_thinking_20251015``, which Claude Code + sends on every request) are LiteLLM-internal: thinking is injected + separately via ``_ensure_thinking_for_clear_thinking_context_management``, + and forwarding the raw edit would trip Bedrock's + ``"context_management: Extra inputs are not permitted"`` 400. + + Refs: + * https://github.com/BerriAI/litellm/issues/27532 + * https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md """ cm = anthropic_messages_request.get("context_management") if not isinstance(cm, dict): @@ -516,15 +535,17 @@ class AmazonAnthropicClaudeMessagesConfig( anthropic_messages_request.pop("context_management", None) return - compact_edits = [e for e in edits if isinstance(e, dict) and e.get("type") == "compact_20260112"] - if compact_edits: - beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - anthropic_messages_request["context_management"] = { - **cm, - "edits": compact_edits, - } - else: + supported = AmazonAnthropicClaudeMessagesConfig._BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS + retained_edits = [e for e in edits if isinstance(e, dict) and e.get("type") in supported] + if not retained_edits: anthropic_messages_request.pop("context_management", None) + return + + beta_set.update(supported[e["type"]] for e in retained_edits) + anthropic_messages_request["context_management"] = { + **cm, + "edits": retained_edits, + } def _get_bedrock_invoke_anthropic_beta_headers( self, diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index d7a62aae38b..532c6ff3598 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -2090,3 +2090,174 @@ def test_bedrock_clear_thinking_leaves_enabled_thinking_on_non_adaptive_model(): assert changed is False assert request["thinking"] == {"type": "enabled", "budget_tokens": 8000} assert "output_config" not in request + + +@pytest.fixture +def local_beta_headers_config(monkeypatch): + from litellm.anthropic_beta_headers_manager import reload_beta_headers_config + + monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True") + reload_beta_headers_config() + yield + monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) + reload_beta_headers_config() + + +def test_bedrock_messages_preserves_clear_tool_uses_context_management_and_adds_beta( + local_beta_headers_config, +): + """ + LIT-3393: Bedrock InvokeModel supports automatic tool-call clearing via + ``clear_tool_uses_20250919`` under the ``context-management-2025-06-27`` + beta. Before the LIT-3393 fix, the transformation stripped this edit (only + ``compact_20260112`` survived) AND the beta was filtered out by + ``filter_and_transform_beta_headers`` for ``bedrock``, producing a Bedrock + 400 ``"context_management: Extra inputs are not permitted"``. + + Post-fix, the edit must reach the body and the beta must reach + ``anthropic_beta``. + + AWS docs ("Automatic tool call clearing (Beta)"): + https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] + optional_params = { + "max_tokens": 4096, + "context_management": { + "edits": [{"type": "clear_tool_uses_20250919"}] + }, + } + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-haiku-4-5-20251001-v1:0", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("context_management") == { + "edits": [{"type": "clear_tool_uses_20250919"}] + }, "clear_tool_uses_20250919 edit must reach Bedrock InvokeModel body" + assert "context-management-2025-06-27" in result.get("anthropic_beta", []), ( + "context-management-2025-06-27 beta must reach the InvokeModel body so " + "the tool-call-clearing edit is accepted" + ) + + +def test_bedrock_messages_preserves_mixed_compact_and_clear_tool_uses_edits( + local_beta_headers_config, +): + """ + LIT-3393: a request mixing ``compact_20260112`` and + ``clear_tool_uses_20250919`` must keep BOTH edits and emit BOTH + anthropic-beta values (``compact-2026-01-12`` + ``context-management-2025-06-27``). + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] + optional_params = { + "max_tokens": 4096, + "context_management": { + "edits": [ + {"type": "compact_20260112"}, + {"type": "clear_tool_uses_20250919"}, + ] + }, + } + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-sonnet-4-6-20250929-v1:0", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + cm = result.get("context_management") + assert cm is not None + edit_types = sorted(e.get("type") for e in cm["edits"]) + assert edit_types == ["clear_tool_uses_20250919", "compact_20260112"] + + betas = result.get("anthropic_beta", []) + assert "compact-2026-01-12" in betas + assert "context-management-2025-06-27" in betas + + +def test_bedrock_messages_filters_clear_thinking_keeps_clear_tool_uses( + local_beta_headers_config, +): + """ + LIT-3393: ``clear_thinking_20251015`` remains LiteLLM-internal (consumed via + thinking-injection) and MUST be stripped from the body, while + ``clear_tool_uses_20250919`` (officially supported on Bedrock InvokeModel) + survives in the same request. + """ + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi"}]}] + optional_params = { + "max_tokens": 4096, + "context_management": { + "edits": [ + {"type": "clear_thinking_20251015", "keep": "all"}, + {"type": "clear_tool_uses_20250919"}, + ] + }, + } + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-7", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + cm = result.get("context_management") + assert cm is not None + assert [e.get("type") for e in cm["edits"]] == [ + "clear_tool_uses_20250919" + ], "clear_thinking_20251015 must still be stripped (LiteLLM-internal)" + + betas = result.get("anthropic_beta", []) + assert "context-management-2025-06-27" in betas + # ``compact-2026-01-12`` was not requested. + assert "compact-2026-01-12" not in betas + + +def test_filter_and_transform_beta_headers_passes_context_management_for_bedrock( + local_beta_headers_config, +): + """ + LIT-3393: ``anthropic_beta_headers_config.json`` previously mapped + ``bedrock.context-management-2025-06-27`` to ``null``, so + ``filter_and_transform_beta_headers`` dropped the header even when the + transformation tried to set it. This regression guard locks the bundled + mapping in place. + + Pinned to the bundled local config via ``LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS`` + so the assertion is not subject to whatever the upstream remote currently + serves or what previous tests left in the module cache. + """ + from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers + + out = filter_and_transform_beta_headers( + ["context-management-2025-06-27"], + provider="bedrock", + ) + assert out == ["context-management-2025-06-27"] + + # Bedrock_converse genuinely lacks it per AWS docs; this guard prevents + # an accidental flip there. + out_converse = filter_and_transform_beta_headers( + ["context-management-2025-06-27"], + provider="bedrock_converse", + ) + assert out_converse == [] + From 1fa200123fa54f5012fcfe46f8a1a9bd8365e58a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:37:49 -0700 Subject: [PATCH 074/101] fix(tests): stop DATABASE_URL env pollution from read-replica tests breaking DB e2e tests (#32653) --- tests/test_litellm/proxy/db/conftest.py | 65 +++++++++++++++++++ .../proxy/db/test_db_url_settings.py | 26 ++++---- .../proxy/db/test_rds_iam_token_expiry.py | 44 ++++--------- .../proxy/db/test_routing_prisma_wrapper.py | 6 +- 4 files changed, 89 insertions(+), 52 deletions(-) create mode 100644 tests/test_litellm/proxy/db/conftest.py diff --git a/tests/test_litellm/proxy/db/conftest.py b/tests/test_litellm/proxy/db/conftest.py new file mode 100644 index 00000000000..a0fb6bed4fa --- /dev/null +++ b/tests/test_litellm/proxy/db/conftest.py @@ -0,0 +1,65 @@ +import os +from collections.abc import Generator +from typing import Optional + +import pytest + +DB_ENV_KEYS = ( + "IAM_TOKEN_DB_AUTH", + "DATABASE_URL", + "DIRECT_URL", + "DATABASE_URL_READ_REPLICA", + "DATABASE_HOST", + "DATABASE_PORT", + "DATABASE_USER", + "DATABASE_USERNAME", + "DATABASE_NAME", + "DATABASE_SCHEMA", + "DATABASE_PASSWORD", + "DATABASE_HOST_READ_REPLICA", + "DATABASE_PORT_READ_REPLICA", + "DATABASE_USER_READ_REPLICA", + "DATABASE_USERNAME_READ_REPLICA", + "DATABASE_NAME_READ_REPLICA", + "DATABASE_SCHEMA_READ_REPLICA", + "DATABASE_PASSWORD_READ_REPLICA", +) + +_db_env_snapshot_key = pytest.StashKey[dict[str, Optional[str]]]() + + +def _db_env_snapshot() -> dict[str, Optional[str]]: + return {key: os.environ.get(key) for key in DB_ENV_KEYS} + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_setup(item: pytest.Item) -> Generator[None, None, None]: + item.stash[_db_env_snapshot_key] = _db_env_snapshot() + return (yield) + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_teardown(item: pytest.Item, nextitem: Optional[pytest.Item]) -> Generator[None, None, None]: + result = yield + before = item.stash[_db_env_snapshot_key] + leaked = {key: value for key, value in _db_env_snapshot().items() if value != before[key]} + for key, original in before.items(): + if original is None: + os.environ.pop(key, None) + else: + os.environ[key] = original + assert not leaked, ( + f"{item.nodeid} leaked DB env vars past monkeypatch teardown: {leaked}. " + "Product code under test writes DATABASE_URL(_READ_REPLICA) into os.environ as a side effect; " + "monkeypatch only restores keys it has a record for, so a value written to a previously unset " + "key survives the test and poisons every later test in this pytest-xdist worker process " + "(DB-backed e2e tests arm themselves on DATABASE_URL and then fail to connect). " + "Use the unset_database_url fixture (or monkeypatch.setenv) so restoration is registered." + ) + return result + + +@pytest.fixture +def unset_database_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DATABASE_URL", "about-to-be-unset") + monkeypatch.delenv("DATABASE_URL") diff --git a/tests/test_litellm/proxy/db/test_db_url_settings.py b/tests/test_litellm/proxy/db/test_db_url_settings.py index 573bd5ae584..e5aa09addab 100644 --- a/tests/test_litellm/proxy/db/test_db_url_settings.py +++ b/tests/test_litellm/proxy/db/test_db_url_settings.py @@ -51,25 +51,21 @@ _MANAGED_DB_ENV_VARS = ( @pytest.fixture(autouse=True) -def _scrub_db_env(): +def _scrub_db_env(monkeypatch): """Start each test from a clean slate and restore the original env afterward. - ``apply_to_env`` writes ``DATABASE_URL`` straight into ``os.environ``, which - ``monkeypatch`` cannot undo. Snapshotting and restoring here keeps a - synthesized URL (e.g. ``writer.example.com``) from leaking into later tests - that read ``DATABASE_URL`` to decide whether to hit a real database. + ``apply_to_env`` writes ``DATABASE_URL`` straight into ``os.environ``. + Registering a setenv+delenv pair per var gives ``monkeypatch`` a restore + record even for previously unset keys, so a synthesized URL (e.g. + ``writer.example.com``) cannot leak into later tests that read + ``DATABASE_URL`` to decide whether to hit a real database. Restoring via + the same ``monkeypatch`` instance the tests use also keeps undo ordering + consistent (a hand-rolled snapshot/restore runs before ``monkeypatch``'s + own undo and gets clobbered by it). """ - saved = {var: os.environ.get(var) for var in _MANAGED_DB_ENV_VARS} for var in _MANAGED_DB_ENV_VARS: - os.environ.pop(var, None) - try: - yield - finally: - for var, value in saved.items(): - if value is None: - os.environ.pop(var, None) - else: - os.environ[var] = value + monkeypatch.setenv(var, "scrubbed") + monkeypatch.delenv(var) def _stub_iam_token(token: str = "FAKE_TOKEN"): diff --git a/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py b/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py index b92fd86ed7a..ca24f856022 100644 --- a/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py +++ b/tests/test_litellm/proxy/db/test_rds_iam_token_expiry.py @@ -26,25 +26,13 @@ class TestPrismaWrapperTokenRefresh: """Tests for the PrismaWrapper RDS IAM token refresh implementation.""" @pytest.fixture - def setup_env(self): + def setup_env(self, monkeypatch, unset_database_url): """Setup environment variables for testing.""" - os.environ["DATABASE_HOST"] = "test-host.rds.amazonaws.com" - os.environ["DATABASE_PORT"] = "5432" - os.environ["DATABASE_USER"] = "test_user" - os.environ["DATABASE_NAME"] = "test_db" - os.environ["IAM_TOKEN_DB_AUTH"] = "True" - yield - # Cleanup - for key in [ - "DATABASE_HOST", - "DATABASE_PORT", - "DATABASE_USER", - "DATABASE_NAME", - "DATABASE_URL", - "IAM_TOKEN_DB_AUTH", - "DATABASE_SCHEMA", - ]: - os.environ.pop(key, None) + monkeypatch.setenv("DATABASE_HOST", "test-host.rds.amazonaws.com") + monkeypatch.setenv("DATABASE_PORT", "5432") + monkeypatch.setenv("DATABASE_USER", "test_user") + monkeypatch.setenv("DATABASE_NAME", "test_db") + monkeypatch.setenv("IAM_TOKEN_DB_AUTH", "True") def _generate_mock_token(self, expires_in_seconds: int = 900) -> str: """Generate a mock IAM token with expiration info.""" @@ -172,22 +160,12 @@ class TestBackgroundRefreshLoop: """Tests for the background refresh loop timing.""" @pytest.fixture - def setup_env(self): + def setup_env(self, monkeypatch, unset_database_url): """Setup environment variables for testing.""" - os.environ["DATABASE_HOST"] = "test-host.rds.amazonaws.com" - os.environ["DATABASE_PORT"] = "5432" - os.environ["DATABASE_USER"] = "test_user" - os.environ["DATABASE_NAME"] = "test_db" - yield - # Cleanup - for key in [ - "DATABASE_HOST", - "DATABASE_PORT", - "DATABASE_USER", - "DATABASE_NAME", - "DATABASE_URL", - ]: - os.environ.pop(key, None) + monkeypatch.setenv("DATABASE_HOST", "test-host.rds.amazonaws.com") + monkeypatch.setenv("DATABASE_PORT", "5432") + monkeypatch.setenv("DATABASE_USER", "test_user") + monkeypatch.setenv("DATABASE_NAME", "test_db") @pytest.mark.asyncio async def test_calculate_seconds_fallback_when_no_url(self, setup_env): diff --git a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py index 92043f44ca9..e5bb8b99507 100644 --- a/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py +++ b/tests/test_litellm/proxy/db/test_routing_prisma_wrapper.py @@ -626,7 +626,7 @@ async def test_getattr_does_not_block_inside_running_loop_on_expired_token(monke assert refresh_calls["count"] == 1 -def test_writer_get_rds_iam_token_defaults_port_when_unset(monkeypatch): +def test_writer_get_rds_iam_token_defaults_port_when_unset(monkeypatch, unset_database_url): """When DATABASE_PORT is unset, the writer must default to the Postgres standard port instead of passing `None` through. Passing None to `generate_iam_auth_token` makes botocore embed the literal string @@ -639,7 +639,6 @@ def test_writer_get_rds_iam_token_defaults_port_when_unset(monkeypatch): monkeypatch.setenv("DATABASE_USER", "litellm") monkeypatch.setenv("DATABASE_NAME", "litellm") monkeypatch.delenv("DATABASE_SCHEMA", raising=False) - monkeypatch.delenv("DATABASE_URL", raising=False) captured: Dict[str, Any] = {} @@ -661,7 +660,7 @@ def test_writer_get_rds_iam_token_defaults_port_when_unset(monkeypatch): assert ":5432/litellm" in (new_url or "") -def test_writer_get_rds_iam_token_uses_database_host_env_vars(monkeypatch): +def test_writer_get_rds_iam_token_uses_database_host_env_vars(monkeypatch, unset_database_url): """Writer's IAM path (no iam_endpoint configured) reads host/port/user/db from the legacy DATABASE_HOST/PORT/USER/NAME env vars and writes the URL back to DATABASE_URL — this is the pre-read-replica behavior the patch @@ -673,7 +672,6 @@ def test_writer_get_rds_iam_token_uses_database_host_env_vars(monkeypatch): monkeypatch.setenv("DATABASE_USER", "litellm") monkeypatch.setenv("DATABASE_NAME", "litellm") monkeypatch.setenv("DATABASE_SCHEMA", "public") - monkeypatch.delenv("DATABASE_URL", raising=False) captured: Dict[str, Any] = {} From 65d0dcfb821adcc80a1e78dc48e37df71f6eda89 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 15:02:05 -0700 Subject: [PATCH 075/101] fix(mcp): never forward an Authorization header that satisfied admission on the tools preview Authorization doubles as the admission fallback when x-litellm-api-key is absent, so a caller who authenticated the preview request that way had their LiteLLM key forwarded to the upstream as the oauth2/client-forwarded token. The preview now forwards Authorization only when the primary admission header is present, which is how the dashboard has always sent it; with no primary header there is no upstream token on the request at all. Applies to oauth2 and both client-forwarded modes; parametrized regression test plus the admission header added to the existing extraction tests to mirror the real UI request shape --- .../mcp_server/rest_endpoints.py | 5 +- .../mcp_server/test_rest_endpoints.py | 141 +++++++++--------- 2 files changed, 71 insertions(+), 75 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 21682b4dd3e..cae304bf73a 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1321,12 +1321,15 @@ if MCP_AVAILABLE: if isinstance(credentials, dict): mcp_auth_header = credentials.get("auth_value") + # Authorization doubles as the admission fallback (LITELLM_API_KEY_HEADER_NAME_SECONDARY): + # when the primary x-litellm-api-key header is absent, the Authorization value is the + # caller's LiteLLM key, not an upstream token, and must never be forwarded upstream. oauth2_headers: Optional[Dict[str, str]] = None if new_mcp_server_request.auth_type in { MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate, - }: + } and headers.get(MCPRequestHandler.LITELLM_API_KEY_HEADER_NAME_PRIMARY): oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers) async def _list_tools_operation(client): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 090b4711dc9..465cebcc12c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -37,10 +37,7 @@ def _build_request( body_bytes = body else: body_bytes = b"" - raw_headers = [ - (key.lower().encode("latin-1"), value.encode("latin-1")) - for key, value in headers.items() - ] + raw_headers = [(key.lower().encode("latin-1"), value.encode("latin-1")) for key, value in headers.items()] scope = { "type": "http", "http_version": "1.1", @@ -62,25 +59,18 @@ def _build_request( def _get_route(path: str, method: str): for route in rest_endpoints.router.routes: - if getattr(route, "path", None) == path and method in getattr( - route, "methods", set() - ): + if getattr(route, "path", None) == path and method in getattr(route, "methods", set()): return route raise AssertionError(f"Route {method} {path} not found") def _route_has_dependency(route, dependency) -> bool: - if any( - getattr(dep, "dependency", None) == dependency - for dep in getattr(route, "dependencies", []) - ): + if any(getattr(dep, "dependency", None) == dependency for dep in getattr(route, "dependencies", [])): return True dependant = getattr(route, "dependant", None) if dependant is None: return False - return any( - getattr(dep, "call", None) == dependency for dep in dependant.dependencies - ) + return any(getattr(dep, "call", None) == dependency for dep in dependant.dependencies) class TestExecuteWithMcpClient: @@ -104,9 +94,7 @@ class TestExecuteWithMcpClient: auth_type=MCPAuth.none, ) - result = await rest_endpoints._execute_with_mcp_client( - payload, failing_operation - ) + result = await rest_endpoints._execute_with_mcp_client(payload, failing_operation) assert result["status"] == "error" assert "stack_trace" not in result @@ -267,15 +255,10 @@ class TestExecuteWithMcpClient: assert result["status"] == "ok" # The incoming Authorization must be dropped — extra_headers should # contain no oauth2 headers (only static_headers, which are None here). - assert ( - captured["extra_headers"] is None - or "Authorization" not in captured["extra_headers"] - ) + assert captured["extra_headers"] is None or "Authorization" not in captured["extra_headers"] @pytest.mark.asyncio - async def test_interactive_oauth_resolves_forwarded_token_via_presented_store( - self, monkeypatch - ): + async def test_interactive_oauth_resolves_forwarded_token_via_presented_store(self, monkeypatch): """Interactive authorization_code preview (oauth2, no client credentials): the forwarded just-authorized token is resolved THROUGH the v2 resolver via a one-shot presented store (cred_provider), not the caller-override path. The bare token (Bearer stripped) is the @@ -433,9 +416,7 @@ class TestExecuteWithMcpClient: return None async def fake_create_client(*args, **kwargs): - raise BaseExceptionGroup( - "test group", [RuntimeError("Cancelled via cancel scope")] - ) + raise BaseExceptionGroup("test group", [RuntimeError("Cancelled via cancel scope")]) monkeypatch.setattr( rest_endpoints.global_mcp_server_manager, @@ -497,9 +478,7 @@ class TestTestToolsList: "message": "Successfully retrieved tools", } - monkeypatch.setattr( - rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False - ) + monkeypatch.setattr(rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False) oauth_call_counter = {"count": 0} @@ -555,9 +534,7 @@ class TestTestToolsList: "message": "Successfully retrieved tools", } - monkeypatch.setattr( - rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False - ) + monkeypatch.setattr(rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False) oauth_headers = {"Authorization": "Bearer oauth"} oauth_call_counter = {"count": 0} @@ -573,7 +550,7 @@ class TestTestToolsList: raising=False, ) - request = _build_request({"authorization": "Bearer incoming"}) + request = _build_request({"authorization": "Bearer incoming", "x-litellm-api-key": "sk-admission"}) payload = NewMCPServerRequest( server_name="example", url="https://example.com", @@ -627,7 +604,7 @@ class TestTestToolsList: raising=False, ) - request = _build_request({"authorization": "Bearer upstream-token"}) + request = _build_request({"authorization": "Bearer upstream-token", "x-litellm-api-key": "sk-admission"}) payload = NewMCPServerRequest( server_name="example", url="https://example.com", @@ -646,6 +623,48 @@ class TestTestToolsList: assert captured["mcp_auth_header"] is None assert captured["oauth2_headers"] == oauth_headers + @pytest.mark.parametrize("auth_type", [MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate]) + async def test_does_not_forward_authorization_that_satisfied_admission(self, monkeypatch, auth_type): + """Authorization is also the admission fallback: with no x-litellm-api-key on the request, + the Authorization value is the caller's LiteLLM key, so forwarding it would send the + admission credential to the upstream.""" + + captured: dict = {} + + async def fake_execute( + request, + operation, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers=None, + ): + captured["oauth2_headers"] = oauth2_headers + return { + "tools": [], + "error": None, + "message": "Successfully retrieved tools", + } + + monkeypatch.setattr(rest_endpoints, "_execute_with_mcp_client", fake_execute, raising=False) + + request = _build_request({"authorization": "Bearer sk-litellm-admission-key"}) + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=auth_type, + ) + + from litellm.proxy._types import LitellmUserRoles + + result = await rest_endpoints.test_tools_list( + request, + payload, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert result["message"] == "Successfully retrieved tools" + assert captured["oauth2_headers"] is None + class TestListToolsRestAPI: pytestmark = pytest.mark.asyncio @@ -775,9 +794,7 @@ class TestListToolsRestAPI: stub_server = StubServer() captured = {} - async def fake_get_tools( - server, server_auth_header, *args, apply_tool_filters=True, **kwargs - ): + async def fake_get_tools(server, server_auth_header, *args, apply_tool_filters=True, **kwargs): captured["apply_tool_filters"] = apply_tool_filters return ["tool-1"] @@ -825,9 +842,7 @@ class TestListToolsRestAPI: assert captured["apply_tool_filters"] is True @pytest.mark.parametrize("upstream_status", [401, 403]) - async def test_upstream_auth_failure_surfaces_status_and_challenge( - self, monkeypatch, upstream_status - ): + async def test_upstream_auth_failure_surfaces_status_and_challenge(self, monkeypatch, upstream_status): """A single-server pass-through request whose upstream rejects the token must surface the upstream status (401 or 403) plus its WWW-Authenticate challenge, not collapse into a 200 ``unexpected_error`` body.""" @@ -1415,9 +1430,7 @@ class TestListToolsRestAPI: oauth_headers = {"Authorization": "Bearer user-oauth-token"} - async def fake_get_user_oauth_extra_headers( - server, user_api_key_dict, prefetched_creds=None - ): + async def fake_get_user_oauth_extra_headers(server, user_api_key_dict, prefetched_creds=None): return oauth_headers captured = {} @@ -1661,9 +1674,7 @@ class TestGetToolsForSingleServer: pytestmark = pytest.mark.asyncio - async def test_filters_tools_by_object_permission_mcp_tool_permissions( - self, monkeypatch - ): + async def test_filters_tools_by_object_permission_mcp_tool_permissions(self, monkeypatch): """Test that tools are filtered by user_api_key_auth.object_permission.mcp_tool_permissions""" from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable @@ -1826,9 +1837,7 @@ class TestGetToolsForSingleServer: # All tools should be returned assert len(result) == 2 - async def test_no_filtering_when_server_not_in_mcp_tool_permissions( - self, monkeypatch - ): + async def test_no_filtering_when_server_not_in_mcp_tool_permissions(self, monkeypatch): """Test that all tools are returned when server is not in mcp_tool_permissions""" from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable @@ -1881,9 +1890,7 @@ class TestGetToolsForSingleServer: # All tools should be returned since server is not in permissions assert len(result) == 2 - async def test_combines_server_allowed_tools_and_object_permission_filters( - self, monkeypatch - ): + async def test_combines_server_allowed_tools_and_object_permission_filters(self, monkeypatch): """Test that both server.allowed_tools and object_permission.mcp_tool_permissions filters are applied""" from litellm.proxy._experimental.mcp_server.server import MCPServer from litellm.proxy._types import LiteLLM_ObjectPermissionTable @@ -2201,9 +2208,7 @@ class TestPreviewOpenAPITools: "paths": { "/repos/{owner}/{repo}/actions/jobs/{job_id}/logs": { "get": { - "operationId": ( - "actions/download-job-logs-for-workflow-run" - ), + "operationId": ("actions/download-job-logs-for-workflow-run"), "summary": "Download job logs", } }, @@ -2246,9 +2251,7 @@ class TestPreviewOpenAPITools: names = [t["name"] for t in result["tools"]] anthropic_re = re.compile(r"^[a-zA-Z0-9_-]{1,128}$") for name in names: - assert anthropic_re.match( - name - ), f"preview tool name {name!r} violates ^[a-zA-Z0-9_-]+$" + assert anthropic_re.match(name), f"preview tool name {name!r} violates ^[a-zA-Z0-9_-]+$" assert "actions_download-job-logs-for-workflow-run" in names assert "pulls_list-files" in names @@ -2305,9 +2308,7 @@ class TestPreviewOpenAPITools: registered_summary_to_name: dict = {} - def fake_create_tool_function( - path, method, operation, base_url - ): # noqa: ANN001 + def fake_create_tool_function(path, method, operation, base_url): # noqa: ANN001 def _f(): return None @@ -2320,9 +2321,7 @@ class TestPreviewOpenAPITools: ) class _StubRegistry: - def register_tool( - self, name, description, input_schema, handler - ): # noqa: ANN001 + def register_tool(self, name, description, input_schema, handler): # noqa: ANN001 registered_summary_to_name[description] = name monkeypatch.setattr( @@ -2331,9 +2330,7 @@ class TestPreviewOpenAPITools: _StubRegistry(), ) - openapi_to_mcp_generator.register_tools_from_openapi( - spec, base_url="https://example.invalid" - ) + openapi_to_mcp_generator.register_tools_from_openapi(spec, base_url="https://example.invalid") assert preview_summary_to_name == registered_summary_to_name, ( f"preview {preview_summary_to_name} != " @@ -2361,15 +2358,11 @@ class TestConnectionErrorMessage: assert secret not in message def test_connect_error_points_at_reachability(self): - message = rest_endpoints._connection_error_message( - httpx.ConnectError("All connection attempts failed") - ) + message = rest_endpoints._connection_error_message(httpx.ConnectError("All connection attempts failed")) assert "unreachable" in message.lower() def test_timeout_error_message(self): - message = rest_endpoints._connection_error_message( - httpx.ConnectTimeout("timed out") - ) + message = rest_endpoints._connection_error_message(httpx.ConnectTimeout("timed out")) assert "unreachable" in message.lower() def test_http_status_error_includes_status_code(self): From d0f1c38d6a9910f6c0f9130ce00f785e94899f37 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 15:35:28 -0700 Subject: [PATCH 076/101] fix(mcp): log only the origin of the upstream MCP url in tool-call metadata The redacted resource kept the path, but hosted MCP servers routinely embed the credential in the path (for example /mcp/s//mcp), and mcp_tool_call_metadata is readable by a caller who can invoke the tool, so the path leaked the upstream credential into spend logs. Only scheme, host, and port are logged now --- litellm/proxy/_experimental/mcp_server/server.py | 11 ++++++----- .../_experimental/mcp_server/test_mcp_server.py | 12 +++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 938cc2bc43b..3550237dd65 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -107,11 +107,12 @@ _MCP_ROUTING_PEEK_MAX_BYTES = 4096 def _redact_mcp_resource_url(url: Optional[str]) -> Optional[str]: - """Reduce an MCP server URL to its bare resource identifier for logging. + """Reduce an MCP server URL to its origin (scheme + host + port) for logging. - Keeps scheme, host, and path; drops userinfo (``user:pass@``), the query string, - and the fragment, so an upstream URL carrying an embedded token, userinfo, or a - secret query parameter never reaches spend-log metadata or logging callbacks. + Everything else is dropped: userinfo (``user:pass@``), the query string, the + fragment, and the path, because hosted MCP servers routinely embed the + credential in the path (e.g. ``/mcp/s/``) and this value is persisted + in spend-log metadata that a caller who can invoke the tool can read back. Returns None when the URL has no host to identify (nothing safe to log). """ if not isinstance(url, str) or not url: @@ -123,7 +124,7 @@ def _redact_mcp_resource_url(url: Optional[str]) -> Optional[str]: if not parts.hostname: return None netloc = f"{parts.hostname}:{parts.port}" if parts.port else parts.hostname - return urlunsplit((parts.scheme, netloc, parts.path, "", "")) or None + return urlunsplit((parts.scheme, netloc, "", "", "")) or None def _invalidate_byok_cred_cache(user_id: str, server_id: str) -> None: diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 0db36e75e48..bba0eb31cfb 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -6859,11 +6859,13 @@ async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow(): @pytest.mark.parametrize( "url, expected", [ - # userinfo + secret query param must both be stripped from the logged resource - ("https://user:s3cr3t@mcp.example.com/mcp?token=abcd1234&x=1", "https://mcp.example.com/mcp"), - ("https://mcp.example.com/mcp#frag", "https://mcp.example.com/mcp"), - ("https://host:8443/a/b?q=1", "https://host:8443/a/b"), - ("https://mcp.notion.com/mcp", "https://mcp.notion.com/mcp"), + # only the origin may be logged: userinfo, query, fragment, and the PATH are all stripped, + # because hosted MCP servers routinely embed the credential in the path (e.g. /mcp/s/) + ("https://user:s3cr3t@mcp.example.com/mcp?token=abcd1234&x=1", "https://mcp.example.com"), + ("https://mcp.example.com/mcp#frag", "https://mcp.example.com"), + ("https://host:8443/a/b?q=1", "https://host:8443"), + ("https://mcp.zapier.com/api/mcp/s/NDgzcret-token/mcp", "https://mcp.zapier.com"), + ("https://mcp.notion.com/mcp", "https://mcp.notion.com"), (None, None), ("", None), ("not a url", None), From d4e02ac047565ed3c14e710a991436f369a08509 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 15:35:28 -0700 Subject: [PATCH 077/101] refactor(mcp): share _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES in the relay gate and tools preview The gateway authorize/token/register gate and the preview header extraction each carried their own inline copy of the oauth2 + client-forwarded mode set, which could drift from the discovery constant the registry builders use; all three surfaces mean the same thing (modes that run the upstream OAuth browser flow), so they now read the one constant --- .../_experimental/mcp_server/discoverable_endpoints.py | 6 +++++- litellm/proxy/_experimental/mcp_server/rest_endpoints.py | 9 ++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 7606deac241..4fd47a97066 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -473,7 +473,11 @@ def _raise_if_not_oauth2(mcp_server: MCPServer) -> None: token is upstream-audienced and held by the caller; the gateway persists nothing for these modes (DCR persistence is opt-in and never enabled on this path). """ - if mcp_server.auth_type in (MCPAuth.oauth2, MCPAuth.true_passthrough, MCPAuth.oauth_delegate): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load + _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, + ) + + if mcp_server.auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: return raise HTTPException( status_code=400, diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index cae304bf73a..74f0b488d20 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -69,6 +69,7 @@ if MCP_AVAILABLE: from mcp.types import Tool as MCPTool from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES, global_mcp_server_manager, ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( @@ -1325,11 +1326,9 @@ if MCP_AVAILABLE: # when the primary x-litellm-api-key header is absent, the Authorization value is the # caller's LiteLLM key, not an upstream token, and must never be forwarded upstream. oauth2_headers: Optional[Dict[str, str]] = None - if new_mcp_server_request.auth_type in { - MCPAuth.oauth2, - MCPAuth.true_passthrough, - MCPAuth.oauth_delegate, - } and headers.get(MCPRequestHandler.LITELLM_API_KEY_HEADER_NAME_PRIMARY): + if new_mcp_server_request.auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES and headers.get( + MCPRequestHandler.LITELLM_API_KEY_HEADER_NAME_PRIMARY + ): oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers) async def _list_tools_operation(client): From 05f39bf9427290e077a6ef07c6d964cc90ef44df Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 10:43:13 -0700 Subject: [PATCH 078/101] fix(mcp): invalidate a browser-authorized upstream token when a mint-relevant field changes An admin who ran Authorize & Fetch and then changed a field that determines which upstream OAuth token gets minted kept using the stale token for tool preview, sessionStorage, and (on the backend) the stored per-user credential and its cache. Grounded in RFC 8707/8693 and the MCP auth spec, a token is bound to one tuple: resource/audience (url), OAuth mode/grant (auth_type, oauth_flow_type), the authorization-server endpoints, and the OAuth client + scopes. A shared getOAuthAuthorizationIdentity captures exactly those fields; transport (http/sse on the same url is the same audience) and delegate_auth_to_upstream (a downstream-usage toggle never sent to the authorize request) are excluded. UI: both the create and edit forms now discard the held token (React state / sessionStorage / hook, plus the fetched token + DCR client in form.credentials) whenever the identity diverges from the one it was authorized against, re-applying the admin's in-flight edit so it is never wiped. The check lives in one shared helper so the two forms cannot drift. Backend: editing an MCP server now compares the pre/post identity and, on a mint-relevant change, purges every stored per-user OAuth credential for the server (DB row + per-user token cache) so no user forwards a token minted for a resource/AS/client that no longer matches. Best-effort; a purge failure never fails the update. --- litellm/proxy/_experimental/mcp_server/db.py | 44 +++++++++ .../mcp_management_endpoints.py | 29 ++++++ .../mcp_server/test_db_credentials.py | 89 +++++++++++++++++++ .../mcp_tools/create_mcp_server.test.tsx | 38 ++++++++ .../mcp_tools/create_mcp_server.tsx | 67 +++++++------- .../components/mcp_tools/mcp_server_edit.tsx | 53 ++++++++++- .../src/components/mcp_tools/types.tsx | 27 ++++++ 7 files changed, 312 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 10081ce19de..8c5e728d86b 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1070,6 +1070,50 @@ async def list_user_oauth_credentials( return results +def mcp_oauth_token_identity(server: Any) -> tuple[Any, ...]: + """The upstream-OAuth-token-determining fields of an MCP server: the resource/audience (url), the + OAuth mode/grant (auth_type, oauth2_flow), the authorization-server endpoints, and the OAuth client + + scopes. Mirrors the dashboard's getOAuthAuthorizationIdentity. When any of these change on a server + update, previously stored per-user tokens were minted for the old identity and are stale. Excludes + transport and delegate_auth_to_upstream, which do not affect what token is minted (RFC 8707/8693).""" + creds = getattr(server, "credentials", None) + creds_dict: Dict[str, Any] = creds if isinstance(creds, dict) else {} + return ( + getattr(server, "url", None), + getattr(server, "auth_type", None), + getattr(server, "oauth2_flow", None), + getattr(server, "authorization_url", None), + getattr(server, "token_url", None), + getattr(server, "registration_url", None), + creds_dict.get("client_id"), + creds_dict.get("client_secret"), + creds_dict.get("scopes"), + ) + + +async def purge_user_oauth_credentials_for_server(prisma_client: PrismaClient, server_id: str) -> int: + """Delete every stored per-user OAuth credential for a server and drop each from the per-user token + cache, so no user keeps a token minted for a superseded configuration. Called when a server update + changes a mint-relevant field (see mcp_oauth_token_identity). Returns the number of rows removed.""" + repo = MCPUserCredentialsRepository(prisma_client) + rows = await repo.table.find_many(where={"server_id": server_id}) + if not rows: + return 0 + await repo.table.delete_many(where={"server_id": server_id}) + from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + mcp_per_user_token_cache, + ) + + for row in rows: + try: + await mcp_per_user_token_cache.delete(row.user_id, server_id) + except Exception as exc: # noqa: BLE001 - cache drop is best-effort; the DB delete is authoritative + verbose_proxy_logger.warning( + "Failed to drop cached MCP OAuth token for user=%s server=%s: %s", row.user_id, server_id, exc + ) + return len(rows) + + async def refresh_user_oauth_token( prisma_client: PrismaClient, user_id: str, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index c9952b245c7..8f6779b17b9 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -125,7 +125,9 @@ if MCP_AVAILABLE: get_user_env_vars_bulk, get_user_oauth_credential, list_user_oauth_credentials, + mcp_oauth_token_identity, merge_user_env_vars, + purge_user_oauth_credentials_for_server, reject_mcp_server, store_user_credential, store_user_oauth_credential, @@ -2318,6 +2320,9 @@ if MCP_AVAILABLE: }, ) + # Snapshot the pre-update identity so we can detect a mint-relevant change below. + old_server_record = await get_mcp_server(prisma_client, payload.server_id) + # try to update the mcp server mcp_server_record_updated = await update_mcp_server( prisma_client, @@ -2336,6 +2341,30 @@ if MCP_AVAILABLE: # Ensure registry is up to date by reloading from database await global_mcp_server_manager.reload_servers_from_database() + # If a field that determines which upstream OAuth token gets minted changed (url/audience, OAuth + # mode/grant, authorization-server endpoints, or the OAuth client + scopes), every stored per-user + # token was minted for the old configuration and is stale. Purge them (DB + cache) so the next + # tool call re-authorizes instead of forwarding a token for a resource/AS/client that no longer + # matches. Best-effort: a purge failure must not fail the update, whose primary job already + # succeeded. + if old_server_record is not None and mcp_oauth_token_identity(old_server_record) != mcp_oauth_token_identity( + mcp_server_record_updated + ): + try: + purged = await purge_user_oauth_credentials_for_server(prisma_client, payload.server_id) + if purged: + verbose_logger.info( + "MCP server %s: purged %d stale per-user OAuth token(s) after a mint-relevant config change", + payload.server_id, + purged, + ) + except Exception as exc: # noqa: BLE001 - purge is best-effort; the server update already succeeded + verbose_logger.warning( + "MCP server %s: failed to purge stale per-user OAuth tokens after config change: %s", + payload.server_id, + exc, + ) + # TODO: Enterprise: Finish audit log trail if litellm.store_audit_logs: pass diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 8b8b8a363d5..628f422fbf1 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -11,6 +11,7 @@ keeps a plain-base64 fallback on read so existing rows continue to work. import base64 import json from datetime import datetime, timedelta, timezone +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest @@ -63,6 +64,94 @@ def _legacy_row(payload: str): return row +def _identity_server(**overrides): + base = dict( + url="https://up.example.com/mcp", + auth_type="oauth2", + oauth2_flow="authorization_code", + authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + registration_url="https://idp.example.com/register", + credentials={"client_id": "cid", "client_secret": "csec", "scopes": ["a"]}, + server_name="srv", + description="d", + ) + base.update(overrides) + return SimpleNamespace(**base) + + +@pytest.mark.parametrize( + "overrides", + [ + {"url": "https://other.example.com/mcp"}, + {"auth_type": "oauth_delegate"}, + {"oauth2_flow": "client_credentials"}, + {"authorization_url": "https://other.example.com/authorize"}, + {"token_url": "https://other.example.com/token"}, + {"registration_url": "https://other.example.com/register"}, + {"credentials": {"client_id": "new", "client_secret": "csec", "scopes": ["a"]}}, + {"credentials": {"client_id": "cid", "client_secret": "rotated", "scopes": ["a"]}}, + {"credentials": {"client_id": "cid", "client_secret": "csec", "scopes": ["b"]}}, + ], +) +def test_mcp_oauth_token_identity_changes_on_mint_relevant_fields(overrides): + from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + + assert mcp_oauth_token_identity(_identity_server()) != mcp_oauth_token_identity(_identity_server(**overrides)) + + +@pytest.mark.parametrize( + "overrides", + [ + {"server_name": "renamed"}, + {"description": "changed"}, + ], +) +def test_mcp_oauth_token_identity_stable_on_non_mint_fields(overrides): + from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + + assert mcp_oauth_token_identity(_identity_server()) == mcp_oauth_token_identity(_identity_server(**overrides)) + + +@pytest.mark.asyncio +async def test_purge_user_oauth_credentials_for_server_deletes_rows_and_cache(monkeypatch): + from litellm.proxy._experimental.mcp_server import oauth2_token_cache + from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + + r1 = MagicMock(user_id="alice", server_id="srv-1") + r2 = MagicMock(user_id="bob", server_id="srv-1") + prisma = MagicMock() + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[r1, r2]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock() + + cache_deletes = [] + monkeypatch.setattr( + oauth2_token_cache.mcp_per_user_token_cache, + "delete", + AsyncMock(side_effect=lambda uid, sid: cache_deletes.append((uid, sid))), + ) + + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") + + assert purged == 2 + prisma.db.litellm_mcpusercredentials.delete_many.assert_awaited_once() + assert set(cache_deletes) == {("alice", "srv-1"), ("bob", "srv-1")} + + +@pytest.mark.asyncio +async def test_purge_user_oauth_credentials_for_server_noop_when_empty(): + from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + + prisma = MagicMock() + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock() + + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") + + assert purged == 0 + prisma.db.litellm_mcpusercredentials.delete_many.assert_not_awaited() + + def _stored_value(prisma) -> str: """Pull the credential_b64 value passed to the most recent upsert call.""" call = prisma.db.litellm_mcpusercredentials.upsert.call_args diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index eecbc253b6b..7339420ed68 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -681,6 +681,44 @@ describe("CreateMCPServer", () => { // Asserted in setupOAuthInteractive }); + it("invalidates the held token when the auth mode changes after Authorize & Fetch", async () => { + await setupOAuthInteractive(); + const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "https://a.example.com/mcp" } }); + }); + act(() => { + oauthHook.onTokenReceived?.({ access_token: "tok-a" }, { clientId: "client-a", clientSecret: "secret-a" }); + }); + oauthHook.reset.mockClear(); + + // Switching the Authentication mode changes the OAuth identity, so the held token is discarded. + await selectAntOption("Authentication", "True Passthrough (no LiteLLM auth)"); + + await waitFor(() => expect(oauthHook.reset).toHaveBeenCalled()); + }); + + it("does NOT invalidate the held token when a non-mint field (server name) changes", async () => { + await setupOAuthInteractive(); + const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "https://a.example.com/mcp" } }); + }); + act(() => { + oauthHook.onTokenReceived?.({ access_token: "tok-a" }, { clientId: "client-a", clientSecret: "secret-a" }); + }); + oauthHook.reset.mockClear(); + + const nameInput = document.getElementById("server_name") as HTMLInputElement; + await act(async () => { + fireEvent.change(nameInput, { target: { value: "Renamed_Server" } }); + }); + + // server_name is not part of the OAuth identity, so the held token must survive the edit. + await waitFor(() => expect(screen.getAllByRole("button", { name: "Add MCP Server" }).length).toBeGreaterThan(0)); + expect(oauthHook.reset).not.toHaveBeenCalled(); + }); + it("includes token_validation in payload when token_validation_json is filled with valid JSON", async () => { vi.mocked(networking.createMCPServer).mockResolvedValue({ server_id: "new-server-oauth", diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 0b39add234c..17c7e7c0b9a 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -15,6 +15,7 @@ import { MCP_OAUTH2_FLOW_M2M, MCP_OAUTH2_FLOW_INTERACTIVE, isClientForwardedTokenMode, + getOAuthAuthorizationIdentity, } from "./types"; import OAuthFormFields from "./OAuthFormFields"; import TruePassthroughWarning from "./TruePassthroughWarning"; @@ -99,7 +100,10 @@ const CreateMCPServer: React.FC = ({ const [oauthAccessToken, setOauthAccessToken] = useState(null); const [logoUrl, setLogoUrl] = useState(undefined); const [oauthDocsUrl, setOauthDocsUrl] = useState(null); - const [authorizedUrl, setAuthorizedUrl] = useState(undefined); + // The OAuth authorization identity (see getOAuthAuthorizationIdentity) captured at the moment a token + // was fetched; undefined when no valid token is held. If any mint-relevant field diverges from this, + // the held token is stale and is discarded so the admin must re-authorize. + const [authorizedIdentity, setAuthorizedIdentity] = useState(undefined); // Single hook call shared by MCPConnectionStatus and MCPToolConfiguration to avoid duplicate requests. const { @@ -125,12 +129,6 @@ const CreateMCPServer: React.FC = ({ const isAwsSigV4AuthType = authType === AUTH_TYPE.AWS_SIGV4; const isM2MFlow = isOAuthAuthType && formValues.oauth_flow_type === OAUTH_FLOW.M2M; - const getOAuthAuthorizationTarget = (values: Record): string | undefined => { - const transport = values.transport || transportType; - const target = transport === TRANSPORT.OPENAPI ? values.spec_path : values.url; - return typeof target === "string" ? target : undefined; - }; - const persistCreateUiState = () => { if (typeof window === "undefined") { return; @@ -207,6 +205,7 @@ const CreateMCPServer: React.FC = ({ // and committed to sessionStorage on submit; it must never be written into form.credentials, // which would persist it as server-level credentials on the created server row. Mirrors the // edit form's onTokenReceived early return. + setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true))); NotificationsManager.success( "Token held for this browser session. Tools can now be previewed and configured; nothing will be saved to LiteLLM.", ); @@ -223,7 +222,9 @@ const CreateMCPServer: React.FC = ({ }; form.setFieldsValue({ credentials }); - setAuthorizedUrl(getOAuthAuthorizationTarget(form.getFieldsValue(true))); + // Capture the identity AFTER writing the DCR'd credentials so the held token is not spuriously + // invalidated by its own credential write. + setAuthorizedIdentity(getOAuthAuthorizationIdentity(form.getFieldsValue(true))); NotificationsManager.success( "OAuth authorization successful! Please click 'Create MCP Server' to save the configuration.", @@ -233,13 +234,24 @@ const CreateMCPServer: React.FC = ({ flowSource: "create", }); - const clearAuthorizedOAuthState = (values: Record) => { - form.resetFields(["credentials", "authorization_url", "token_url", "registration_url"]); - form.setFieldsValue(values); + // Discard the held browser-authorized token and its tool preview when the authorization identity + // changes (or the modal closes). For oauth2 the fetched token + DCR client also live in + // form.credentials, and the discovered endpoints in authorization_url/token_url/registration_url, so + // those form fields are reset too; whatever the admin just changed (passed via changedValues) is + // re-applied so the invalidation never wipes their in-flight edit. + const CLEARED_ON_INVALIDATION = ["credentials", "authorization_url", "token_url", "registration_url"] as const; + const clearHeldOAuthToken = (changedValues: Record = {}) => { setOauthAccessToken(null); clearTools(); resetOAuthFlow(); - setAuthorizedUrl(undefined); + setAuthorizedIdentity(undefined); + form.resetFields([...CLEARED_ON_INVALIDATION]); + const preserved = Object.fromEntries( + CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]), + ); + if (Object.keys(preserved).length > 0) { + form.setFieldsValue(preserved); + } }; React.useEffect(() => { @@ -577,7 +589,7 @@ const CreateMCPServer: React.FC = ({ : { spec_path: undefined, command: undefined, args: undefined, env: undefined }; const nextValues = - authorizedUrl === undefined + authorizedIdentity === undefined ? transportValues : { ...transportValues, @@ -587,10 +599,9 @@ const CreateMCPServer: React.FC = ({ registration_url: undefined, }; - if (authorizedUrl !== undefined) { - clearAuthorizedOAuthState(nextValues); - } else { - form.setFieldsValue(nextValues); + form.setFieldsValue(nextValues); + if (authorizedIdentity !== undefined) { + clearHeldOAuthToken(); } }; @@ -652,28 +663,18 @@ const CreateMCPServer: React.FC = ({ setOauthAccessToken(null); clearTools(); resetOAuthFlow(); - setAuthorizedUrl(undefined); + setAuthorizedIdentity(undefined); } }, [isModalVisible, form, clearTools, resetOAuthFlow]); const isAdmin = isAdminRole(userRole); const handleFormValuesChange = (changedValues: Record, allValues: Record) => { - const changedAuthorizationTarget = "url" in changedValues || "spec_path" in changedValues; - if ( - changedAuthorizationTarget && - authorizedUrl !== undefined && - getOAuthAuthorizationTarget(allValues) !== authorizedUrl - ) { - const invalidated = { - credentials: undefined, - authorization_url: changedValues.authorization_url, - token_url: changedValues.token_url, - registration_url: changedValues.registration_url, - }; - clearAuthorizedOAuthState(invalidated); - setFormValues({ ...allValues, ...invalidated }); - return; + // Any change to a mint-relevant field (url, auth_type, oauth_flow_type, client creds/scopes, or the + // authorization/token/registration endpoints — see getOAuthAuthorizationIdentity) makes a held token + // stale, so discard it and force a fresh authorize. + if (authorizedIdentity !== undefined && getOAuthAuthorizationIdentity(allValues) !== authorizedIdentity) { + clearHeldOAuthToken(changedValues); } setFormValues(allValues); }; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index ee7938d2904..55adcf2bb59 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -5,6 +5,7 @@ import { Button, TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/rea import { AUTH_TYPE, isClientForwardedTokenMode, + getOAuthAuthorizationIdentity, OAUTH_FLOW, MCP_OAUTH2_FLOW_M2M, MCP_OAUTH2_FLOW_INTERACTIVE, @@ -15,7 +16,7 @@ import { oauth2FlowToFormValue, } from "./types"; import { updateMCPServer, listMCPTools, storeMCPOAuthUserCredential } from "../networking"; -import { getToken, isTokenValid, setToken } from "@/utils/mcpTokenStore"; +import { getToken, isTokenValid, removeToken, setToken } from "@/utils/mcpTokenStore"; import { buildMcpPassthroughAuthHeader } from "@/utils/mcpHeaderUtils"; import MCPServerCostConfig from "./mcp_server_cost_config"; import MCPPermissionManagement from "./MCPPermissionManagement"; @@ -136,11 +137,17 @@ const MCPServerEdit: React.FC = ({ // that read only mcpServer.auth_type go stale the moment the admin switches modes in the form. const getEffectiveAuthType = () => form.getFieldValue("auth_type") ?? mcpServer.auth_type; + // The OAuth authorization identity (see getOAuthAuthorizationIdentity) captured when a token is fetched + // in this edit session; undefined when none is held. If a mint-relevant field later diverges from it, + // the held token (hook response + sessionStorage) is discarded so the admin must re-authorize. + const authorizedIdentityRef = React.useRef(undefined); + const { startOAuthFlow, status: oauthStatus, error: oauthError, tokenResponse: oauthTokenResponse, + reset: resetOAuthFlow, } = useMcpOAuthFlow({ accessToken, getCredentials: () => form.getFieldValue("credentials"), @@ -183,6 +190,7 @@ const MCPServerEdit: React.FC = ({ return; } + authorizedIdentityRef.current = getOAuthAuthorizationIdentity(form.getFieldsValue(true)); if (isClientForwardedTokenMode(getEffectiveAuthType())) { const browserHeldToken = { access_token: token.access_token, @@ -205,6 +213,8 @@ const MCPServerEdit: React.FC = ({ }; form.setFieldsValue({ credentials }); + // Re-capture after writing credentials so the token is not invalidated by its own credential write. + authorizedIdentityRef.current = getOAuthAuthorizationIdentity(form.getFieldsValue(true)); NotificationsManager.success( "OAuth authorization successful! Please click 'Update MCP Server' to save the credentials.", @@ -378,6 +388,39 @@ const MCPServerEdit: React.FC = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [mcpServer, accessToken, userID, oauthTokenResponse?.access_token]); + // Invalidate a token authorized in this edit session once any mint-relevant field diverges from the + // identity it was minted against (url, auth_type, oauth_flow_type, client creds/scopes, or the + // authorization/token/registration endpoints — see getOAuthAuthorizationIdentity). Discards the hook + // token (resetOAuthFlow, which re-runs fetchTools to prompt a fresh authorize), the sessionStorage + // token (removeToken, browser-held modes), and the fetched token/DCR client in form.credentials + the + // discovered endpoint fields; the admin's in-flight edit is re-applied so it is never wiped. Only fires + // when a token was actually authorized here (ref set), so a token already valid for the saved server on + // mount is left untouched. Driven from onValuesChange (user input only), never programmatic resets. + const CLEARED_ON_INVALIDATION = ["credentials", "authorization_url", "token_url", "registration_url"] as const; + const clearHeldOAuthToken = (changedValues: Record = {}) => { + authorizedIdentityRef.current = undefined; + if (mcpServer.server_id) { + removeToken(mcpServer.server_id, userID); + } + resetOAuthFlow(); + form.resetFields([...CLEARED_ON_INVALIDATION]); + const preserved = Object.fromEntries( + CLEARED_ON_INVALIDATION.filter((key) => key in changedValues).map((key) => [key, changedValues[key]]), + ); + if (Object.keys(preserved).length > 0) { + form.setFieldsValue(preserved); + } + }; + + const handleFormValuesChange = (changedValues: Record) => { + if ( + authorizedIdentityRef.current !== undefined && + getOAuthAuthorizationIdentity(form.getFieldsValue(true)) !== authorizedIdentityRef.current + ) { + clearHeldOAuthToken(changedValues); + } + }; + const fetchTools = async () => { if (!accessToken || !mcpServer.server_id) return; @@ -805,7 +848,13 @@ const MCPServerEdit: React.FC = ({ -

+ sse on the same url is the same audience; a transport switch +// only matters when it changes the target url, which `target` already captures), delegate_auth_to_upstream +// (a downstream-usage toggle that is never sent to the authorize request), and all metadata/RBAC/routing +// fields. Shared by the create and edit forms so their invalidation logic cannot drift. +export const getOAuthAuthorizationIdentity = (values: Record): string => { + const credentials = (values.credentials ?? {}) as Record; + const target = values.transport === TRANSPORT.OPENAPI ? values.spec_path : values.url; + const identity = { + target: typeof target === "string" ? target : null, + auth_type: values.auth_type ?? null, + oauth_flow_type: values.oauth_flow_type ?? null, + client_id: credentials.client_id ?? null, + client_secret: credentials.client_secret ?? null, + scopes: credentials.scopes ?? null, + authorization_url: values.authorization_url ?? null, + token_url: values.token_url ?? null, + registration_url: values.registration_url ?? null, + }; + return JSON.stringify(identity); +}; + // Backend value of `oauth2_flow` that marks a machine-to-machine server. Distinct // from the UI-local OAUTH_FLOW.M2M ("m2m"); this is what the API actually returns. export const MCP_OAUTH2_FLOW_M2M = "client_credentials"; From 48124734a08d40ec1b17c3de86c1af0cafc6fa97 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 12:41:15 -0700 Subject: [PATCH 079/101] fix(mcp): compare the token identity decrypted and invalidate every per-user token store Review follow-ups on the stale-token invalidation. The backend identity now decrypts client_id and client_secret before comparing: the stored values are NaCl-encrypted with a fresh nonce on every write, so comparing ciphertext flagged every routine save as a mint-relevant change and purged per-user tokens that were still valid. The identity also gains spec_path, the audience for OpenAPI servers, and parses credentials stored as a JSON string The purge now routes each (user, server) through the manager's invalidate_user_oauth_token_cache, which becomes the single invalidation point covering both the legacy per-user token cache and the v2 per-user OAuth token store; previously the purge evicted only the legacy cache while the revoke path evicted only the v2 store, so each path left the other cache serving a replaced token until its TTL. A credential row racing in between the find and the delete is now detected via the delete_many count and logged; its cache entry expires by TTL On the dashboard, CLEARED_ON_INVALIDATION and the staleness check move to types.tsx as the single shared implementation for both forms. The edit form's transport handler now rechecks the identity after its programmatic setFieldsValue calls, which antd does not report through onValuesChange, so a token no longer survives a transport switch that clears the mint target. The create form rebuilds formValues from the post-reset form state after an invalidation instead of publishing the pre-reset snapshot, so the tool preview can no longer refetch with the discarded DCR client. Both transport handlers now share the recheck, which also stops the create form from over-invalidating on an http to sse swap that keeps the same url and therefore the same audience --- litellm/proxy/_experimental/mcp_server/db.py | 76 ++++++-- .../mcp_server/mcp_server_manager.py | 17 +- .../mcp_server/test_db_credentials.py | 182 +++++++++--------- .../mcp_server/test_mcp_server_manager.py | 37 +++- .../mcp_tools/create_mcp_server.test.tsx | 50 +++++ .../mcp_tools/create_mcp_server.tsx | 32 ++- .../mcp_tools/mcp_server_edit.test.tsx | 81 +++++++- .../components/mcp_tools/mcp_server_edit.tsx | 21 +- .../src/components/mcp_tools/types.tsx | 15 ++ 9 files changed, 367 insertions(+), 144 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 8c5e728d86b..135a9e055d6 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1070,48 +1070,82 @@ async def list_user_oauth_credentials( return results +def _decrypted_credential_field(creds: Dict[str, Any], field: str) -> Any: + """Return one credential field decrypted with the global salt key; non-string and legacy + plaintext values come back unchanged (decrypt_value_helper returns the original on failure).""" + value = creds.get(field) + if not isinstance(value, str): + return value + return decrypt_value_helper( + value=value, + key=field, + exception_type="debug", + return_original_value=True, + ) + + def mcp_oauth_token_identity(server: Any) -> tuple[Any, ...]: - """The upstream-OAuth-token-determining fields of an MCP server: the resource/audience (url), the - OAuth mode/grant (auth_type, oauth2_flow), the authorization-server endpoints, and the OAuth client + - scopes. Mirrors the dashboard's getOAuthAuthorizationIdentity. When any of these change on a server - update, previously stored per-user tokens were minted for the old identity and are stale. Excludes - transport and delegate_auth_to_upstream, which do not affect what token is minted (RFC 8707/8693).""" + """The upstream-OAuth-token-determining fields of an MCP server: the resource/audience (url, or + spec_path for OpenAPI servers), the OAuth mode/grant (auth_type, oauth2_flow), the + authorization-server endpoints, and the OAuth client + scopes. Mirrors the dashboard's + getOAuthAuthorizationIdentity. When any of these change on a server update, previously stored + per-user tokens were minted for the old identity and are stale. Excludes transport and + delegate_auth_to_upstream, which do not affect what token is minted (RFC 8707/8693). + + client_id/client_secret are compared decrypted: stored values are NaCl-encrypted with a fresh + nonce on every write, so comparing ciphertext would flag every routine save as an identity + change and purge tokens that are still valid.""" creds = getattr(server, "credentials", None) - creds_dict: Dict[str, Any] = creds if isinstance(creds, dict) else {} + if isinstance(creds, str): + try: + parsed: Any = json.loads(creds) + except ValueError: + parsed = None + else: + parsed = creds + creds_dict: Dict[str, Any] = parsed if isinstance(parsed, dict) else {} return ( getattr(server, "url", None), + getattr(server, "spec_path", None), getattr(server, "auth_type", None), getattr(server, "oauth2_flow", None), getattr(server, "authorization_url", None), getattr(server, "token_url", None), getattr(server, "registration_url", None), - creds_dict.get("client_id"), - creds_dict.get("client_secret"), + _decrypted_credential_field(creds_dict, "client_id"), + _decrypted_credential_field(creds_dict, "client_secret"), creds_dict.get("scopes"), ) async def purge_user_oauth_credentials_for_server(prisma_client: PrismaClient, server_id: str) -> int: - """Delete every stored per-user OAuth credential for a server and drop each from the per-user token - cache, so no user keeps a token minted for a superseded configuration. Called when a server update - changes a mint-relevant field (see mcp_oauth_token_identity). Returns the number of rows removed.""" + """Delete every stored per-user OAuth credential for a server and invalidate each user's cached + token everywhere it can be served from (the legacy per-user token cache and the v2 per-user OAuth + token store), so no user keeps a token minted for a superseded configuration. Called when a server + update changes a mint-relevant field (see mcp_oauth_token_identity). Returns the number of rows + removed. A row inserted between the find and the delete is removed from the DB but cannot be + evicted from the caches (its user_id was never seen); that case is detected, logged, and bounded + by the cache TTL.""" repo = MCPUserCredentialsRepository(prisma_client) rows = await repo.table.find_many(where={"server_id": server_id}) if not rows: return 0 - await repo.table.delete_many(where={"server_id": server_id}) - from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( - mcp_per_user_token_cache, + deleted_count = await repo.table.delete_many(where={"server_id": server_id}) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, ) for row in rows: - try: - await mcp_per_user_token_cache.delete(row.user_id, server_id) - except Exception as exc: # noqa: BLE001 - cache drop is best-effort; the DB delete is authoritative - verbose_proxy_logger.warning( - "Failed to drop cached MCP OAuth token for user=%s server=%s: %s", row.user_id, server_id, exc - ) - return len(rows) + await global_mcp_server_manager.invalidate_user_oauth_token_cache(row.user_id, server_id) + if deleted_count != len(rows): + verbose_proxy_logger.warning( + "MCP server %s: purge removed %d credential row(s) but %d were enumerated; " + "row(s) raced in during the purge and their cached tokens will expire by TTL", + server_id, + deleted_count, + len(rows), + ) + return deleted_count async def refresh_user_oauth_token( diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 356ed7a2729..cc4a9e63105 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -57,7 +57,10 @@ from litellm.proxy._experimental.mcp_server.elicitation_handler import ( from litellm.proxy._experimental.mcp_server.sampling_handler import ( MCP_SAMPLING_AVAILABLE, ) -from litellm.proxy._experimental.mcp_server.oauth2_token_cache import resolve_mcp_auth +from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + mcp_per_user_token_cache, + resolve_mcp_auth, +) from litellm.proxy._experimental.mcp_server.outbound_credentials import ( Error, Ok, @@ -4053,10 +4056,13 @@ class MCPServerManager: return await self._cred_provider.has_user_token(to_subject(user_api_key_auth, None), spec) async def invalidate_user_oauth_token_cache(self, user_id: str, server_id: str) -> None: - """Drop the v2 chain's cached token for ``(user_id, server_id)`` after the credential row - changes (re-auth, revoke), so the next resolve reads the new row instead of serving the - replaced token until its cache TTL. Best-effort: a cache-drop failure is logged, never - raised, because the DB write already succeeded and the TTL remains the backstop. + """Drop every cached token for ``(user_id, server_id)`` after the credential row changes + (re-auth, revoke, config-change purge): the v2 chain's cache and the legacy per-user token + cache, so the next resolve reads the new row instead of serving the replaced token until its + cache TTL, whichever path resolves it. This is the single invalidation point for per-user + OAuth tokens; callers must not evict individual caches directly. Best-effort: a cache-drop + failure is logged, never raised, because the DB write already succeeded and the TTL remains + the backstop. """ try: await self._per_user_oauth_token_store.invalidate(user_id, server_id) @@ -4064,6 +4070,7 @@ class MCPServerManager: verbose_logger.warning( "Failed to invalidate cached MCP OAuth token for user=%s server=%s: %s", user_id, server_id, exc ) + await mcp_per_user_token_cache.delete(user_id, server_id) async def _resolve_oauth2_headers_for_tool_call( self, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 628f422fbf1..51641991ef9 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -84,6 +84,7 @@ def _identity_server(**overrides): "overrides", [ {"url": "https://other.example.com/mcp"}, + {"spec_path": "https://up.example.com/openapi.json"}, {"auth_type": "oauth_delegate"}, {"oauth2_flow": "client_credentials"}, {"authorization_url": "https://other.example.com/authorize"}, @@ -113,29 +114,91 @@ def test_mcp_oauth_token_identity_stable_on_non_mint_fields(overrides): assert mcp_oauth_token_identity(_identity_server()) == mcp_oauth_token_identity(_identity_server(**overrides)) +def _encrypted_creds_json(client_id: str = "cid", client_secret: str = "csec") -> str: + from litellm.proxy._experimental.mcp_server.db import encrypt_credentials + + encrypted = encrypt_credentials( + credentials={"client_id": client_id, "client_secret": client_secret, "scopes": ["a"]}, + encryption_key=None, + ) + return json.dumps(encrypted) + + +def test_mcp_oauth_token_identity_stable_across_reencryption(): + """Stored client_id/client_secret are NaCl-encrypted with a fresh nonce on every write, so two + saves of the SAME plaintext produce different ciphertext. The identity must compare decrypted + values; comparing ciphertext would flag every routine save as a mint-relevant change and purge + per-user tokens that are still valid.""" + from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + + first = _encrypted_creds_json() + second = _encrypted_creds_json() + assert first != second + + assert mcp_oauth_token_identity(_identity_server(credentials=first)) == mcp_oauth_token_identity( + _identity_server(credentials=second) + ) + + +def test_mcp_oauth_token_identity_detects_change_under_encryption(): + from litellm.proxy._experimental.mcp_server.db import mcp_oauth_token_identity + + unchanged = _identity_server(credentials=_encrypted_creds_json()) + changed = _identity_server(credentials=_encrypted_creds_json(client_id="other")) + assert mcp_oauth_token_identity(unchanged) != mcp_oauth_token_identity(changed) + + @pytest.mark.asyncio -async def test_purge_user_oauth_credentials_for_server_deletes_rows_and_cache(monkeypatch): - from litellm.proxy._experimental.mcp_server import oauth2_token_cache +async def test_purge_user_oauth_credentials_for_server_invalidates_every_store(monkeypatch): + """The purge must route each (user, server) through the manager's shared invalidation, which is + the single point covering both the legacy per-user token cache and the v2 per-user OAuth token + store; evicting only one cache lets the other keep serving a token minted for the old config.""" + from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server r1 = MagicMock(user_id="alice", server_id="srv-1") r2 = MagicMock(user_id="bob", server_id="srv-1") prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[r1, r2]) - prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock() + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) - cache_deletes = [] + invalidations = [] monkeypatch.setattr( - oauth2_token_cache.mcp_per_user_token_cache, - "delete", - AsyncMock(side_effect=lambda uid, sid: cache_deletes.append((uid, sid))), + mcp_server_manager.global_mcp_server_manager, + "invalidate_user_oauth_token_cache", + AsyncMock(side_effect=lambda uid, sid: invalidations.append((uid, sid))), ) purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") assert purged == 2 prisma.db.litellm_mcpusercredentials.delete_many.assert_awaited_once() - assert set(cache_deletes) == {("alice", "srv-1"), ("bob", "srv-1")} + assert set(invalidations) == {("alice", "srv-1"), ("bob", "srv-1")} + + +@pytest.mark.asyncio +async def test_purge_user_oauth_credentials_for_server_logs_raced_rows(monkeypatch): + from litellm.proxy._experimental.mcp_server import db as db_module + from litellm.proxy._experimental.mcp_server import mcp_server_manager + from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + + prisma = MagicMock() + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock( + return_value=[MagicMock(user_id="alice", server_id="srv-1")] + ) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) + monkeypatch.setattr( + mcp_server_manager.global_mcp_server_manager, + "invalidate_user_oauth_token_cache", + AsyncMock(), + ) + warning = MagicMock() + monkeypatch.setattr(db_module.verbose_proxy_logger, "warning", warning) + + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") + + assert purged == 2 + warning.assert_called_once() @pytest.mark.asyncio @@ -225,9 +288,7 @@ async def test_store_user_oauth_credential_does_not_persist_plaintext(): access_token = "ya29.a0AfH6SMBverysecretaccesstoken" prisma = _make_prisma_with_existing(row=None) - await store_user_oauth_credential( - prisma, "alice", "srv-1", access_token, refresh_token="rfr-xyz" - ) + await store_user_oauth_credential(prisma, "alice", "srv-1", access_token, refresh_token="rfr-xyz") stored = _stored_value(prisma) try: @@ -310,9 +371,7 @@ async def test_byok_guard_rejects_overwriting_encrypted_byok(): encrypted_row = MagicMock() encrypted_row.credential_b64 = _stored_value(prisma) - prisma.db.litellm_mcpusercredentials.find_unique = AsyncMock( - return_value=encrypted_row - ) + prisma.db.litellm_mcpusercredentials.find_unique = AsyncMock(return_value=encrypted_row) with pytest.raises(ValueError, match="could not be verified as an OAuth2"): await store_user_oauth_credential(prisma, "alice", "srv-1", "tok") @@ -354,18 +413,14 @@ async def test_list_oauth_credentials_filters_byok_and_returns_payloads(): "connected_at": "2024-01-01T00:00:00Z", } legacy_row = MagicMock() - legacy_row.credential_b64 = base64.urlsafe_b64encode( - json.dumps(legacy_payload).encode() - ).decode() + legacy_row.credential_b64 = base64.urlsafe_b64encode(json.dumps(legacy_payload).encode()).decode() legacy_row.server_id = "srv-legacy" byok_row = MagicMock() byok_row.credential_b64 = base64.urlsafe_b64encode(b"plain-byok-key").decode() byok_row.server_id = "srv-byok" - prisma.db.litellm_mcpusercredentials.find_many = AsyncMock( - return_value=[encrypted_row, legacy_row, byok_row] - ) + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[encrypted_row, legacy_row, byok_row]) results = await list_user_oauth_credentials(prisma, "alice") @@ -415,9 +470,7 @@ async def test_rotate_re_encrypts_byok_with_new_key(monkeypatch): prisma.db.litellm_mcpusercredentials.update = AsyncMock() new_master_key = "rotated-salt-key-9999-9999-9999-9999" - await rotate_mcp_user_credentials_master_key( - prisma_client=prisma, new_master_key=new_master_key - ) + await rotate_mcp_user_credentials_master_key(prisma_client=prisma, new_master_key=new_master_key) update_call = prisma.db.litellm_mcpusercredentials.update.call_args new_stored = update_call.kwargs["data"]["credential_b64"] @@ -445,19 +498,13 @@ async def test_rotate_migrates_legacy_plaintext_rows(monkeypatch): legacy_row.user_id = "alice" legacy_row.server_id = "srv-legacy" legacy_row.credential_b64 = base64.urlsafe_b64encode(b"legacy-plain").decode() - prisma.db.litellm_mcpusercredentials.find_many = AsyncMock( - return_value=[legacy_row] - ) + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[legacy_row]) prisma.db.litellm_mcpusercredentials.update = AsyncMock() new_key = "another-rotation-key-aaaa-bbbb-cccc-dddd" - await rotate_mcp_user_credentials_master_key( - prisma_client=prisma, new_master_key=new_key - ) + await rotate_mcp_user_credentials_master_key(prisma_client=prisma, new_master_key=new_key) - new_stored = prisma.db.litellm_mcpusercredentials.update.call_args.kwargs["data"][ - "credential_b64" - ] + new_stored = prisma.db.litellm_mcpusercredentials.update.call_args.kwargs["data"]["credential_b64"] monkeypatch.setenv("LITELLM_SALT_KEY", new_key) assert ( decrypt_value_helper( @@ -485,14 +532,10 @@ async def test_rotate_skips_undecodable_rows(): good_row.server_id = "srv-ok" good_row.credential_b64 = base64.urlsafe_b64encode(b"good-byok").decode() - prisma.db.litellm_mcpusercredentials.find_many = AsyncMock( - return_value=[bad_row, good_row] - ) + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[bad_row, good_row]) prisma.db.litellm_mcpusercredentials.update = AsyncMock() - await rotate_mcp_user_credentials_master_key( - prisma_client=prisma, new_master_key="new-key-xxxx" - ) + await rotate_mcp_user_credentials_master_key(prisma_client=prisma, new_master_key="new-key-xxxx") # Only one update call — the good row. assert prisma.db.litellm_mcpusercredentials.update.call_count == 1 @@ -508,9 +551,7 @@ def _oauth_cred(access_token="at-live", refresh_token=None, expires_in_seconds=N if refresh_token is not None: cred["refresh_token"] = refresh_token if expires_in_seconds is not None: - cred["expires_at"] = ( - datetime.now(timezone.utc) + timedelta(seconds=expires_in_seconds) - ).isoformat() + cred["expires_at"] = (datetime.now(timezone.utc) + timedelta(seconds=expires_in_seconds)).isoformat() return cred @@ -527,12 +568,7 @@ def test_expiry_buffer_treats_soon_to_expire_as_expired(): cred = _oauth_cred(expires_in_seconds=30) assert is_oauth_credential_expired(cred, buffer_seconds=60) is True # A token comfortably beyond the buffer stays valid. - assert ( - is_oauth_credential_expired( - _oauth_cred(expires_in_seconds=600), buffer_seconds=60 - ) - is False - ) + assert is_oauth_credential_expired(_oauth_cred(expires_in_seconds=600), buffer_seconds=60) is False def test_expiry_past_is_expired_regardless_of_buffer(): @@ -554,9 +590,7 @@ async def test_resolve_returns_valid_token_without_refreshing(monkeypatch): refresh = AsyncMock() monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) - cred = _oauth_cred( - access_token="at-live", refresh_token="rt-1", expires_in_seconds=600 - ) + cred = _oauth_cred(access_token="at-live", refresh_token="rt-1", expires_in_seconds=600) result = await resolve_valid_user_oauth_token( user_id="alice", server=MagicMock(), cred=cred, prisma_client=MagicMock() ) @@ -572,15 +606,11 @@ async def test_resolve_refreshes_expired_token_with_refresh_token(monkeypatch): # new token rather than returning None (which left the UI tool list empty). import litellm.proxy._experimental.mcp_server.db as db_mod - refreshed = _oauth_cred( - access_token="at-fresh", refresh_token="rt-2", expires_in_seconds=3600 - ) + refreshed = _oauth_cred(access_token="at-fresh", refresh_token="rt-2", expires_in_seconds=3600) refresh = AsyncMock(return_value=refreshed) monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) - expired = _oauth_cred( - access_token="at-dead", refresh_token="rt-1", expires_in_seconds=-5 - ) + expired = _oauth_cred(access_token="at-dead", refresh_token="rt-1", expires_in_seconds=-5) result = await resolve_valid_user_oauth_token( user_id="alice", server=MagicMock(), cred=expired, prisma_client=MagicMock() ) @@ -599,9 +629,7 @@ async def test_resolve_refreshes_token_expiring_within_buffer(monkeypatch): refresh = AsyncMock(return_value=refreshed) monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) - soon = _oauth_cred( - access_token="at-soon", refresh_token="rt-1", expires_in_seconds=30 - ) + soon = _oauth_cred(access_token="at-soon", refresh_token="rt-1", expires_in_seconds=30) result = await resolve_valid_user_oauth_token( user_id="alice", server=MagicMock(), cred=soon, prisma_client=MagicMock() ) @@ -635,9 +663,7 @@ async def test_resolve_returns_none_when_refresh_fails(monkeypatch): refresh = AsyncMock(return_value=None) monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) - expired = _oauth_cred( - access_token="at-dead", refresh_token="rt-1", expires_in_seconds=-5 - ) + expired = _oauth_cred(access_token="at-dead", refresh_token="rt-1", expires_in_seconds=-5) result = await resolve_valid_user_oauth_token( user_id="alice", server=MagicMock(), cred=expired, prisma_client=MagicMock() ) @@ -654,9 +680,7 @@ async def test_resolve_returns_none_for_missing_credential(monkeypatch): monkeypatch.setattr(db_mod, "refresh_user_oauth_token", refresh) assert ( - await resolve_valid_user_oauth_token( - user_id="alice", server=MagicMock(), cred=None, prisma_client=MagicMock() - ) + await resolve_valid_user_oauth_token(user_id="alice", server=MagicMock(), cred=None, prisma_client=MagicMock()) is None ) assert ( @@ -690,19 +714,13 @@ async def test_rotate_user_env_vars_re_encrypts_with_new_key(monkeypatch): encrypted_old = encrypt_value_helper(json.dumps(values)) prisma = MagicMock() - prisma.db.litellm_mcpuserenvvars.find_many = AsyncMock( - return_value=[_env_var_row(encrypted_old)] - ) + prisma.db.litellm_mcpuserenvvars.find_many = AsyncMock(return_value=[_env_var_row(encrypted_old)]) prisma.db.litellm_mcpuserenvvars.update = AsyncMock() new_master_key = "rotated-env-key-1111-2222-3333-4444" - await rotate_mcp_user_env_vars_master_key( - prisma_client=prisma, new_master_key=new_master_key - ) + await rotate_mcp_user_env_vars_master_key(prisma_client=prisma, new_master_key=new_master_key) - new_stored = prisma.db.litellm_mcpuserenvvars.update.call_args.kwargs["data"][ - "values_b64" - ] + new_stored = prisma.db.litellm_mcpuserenvvars.update.call_args.kwargs["data"]["values_b64"] assert new_stored != encrypted_old, "rotation must produce different ciphertext" monkeypatch.setenv("LITELLM_SALT_KEY", new_master_key) @@ -719,18 +737,14 @@ async def test_rotate_user_env_vars_re_encrypts_with_new_key(monkeypatch): async def test_rotate_user_env_vars_skips_undecryptable_rows(): # A corrupt row must be skipped (not overwritten) so recoverable data is # preserved and one bad row does not abort the rest of the rotation. - good = _env_var_row( - encrypt_value_helper(json.dumps({"A": "1"})), server_id="srv-ok" - ) + good = _env_var_row(encrypt_value_helper(json.dumps({"A": "1"})), server_id="srv-ok") bad = _env_var_row("!!! not encrypted !!!", server_id="srv-corrupt") prisma = MagicMock() prisma.db.litellm_mcpuserenvvars.find_many = AsyncMock(return_value=[bad, good]) prisma.db.litellm_mcpuserenvvars.update = AsyncMock() - await rotate_mcp_user_env_vars_master_key( - prisma_client=prisma, new_master_key="new-key-xxxx" - ) + await rotate_mcp_user_env_vars_master_key(prisma_client=prisma, new_master_key="new-key-xxxx") assert prisma.db.litellm_mcpuserenvvars.update.call_count == 1 where = prisma.db.litellm_mcpuserenvvars.update.call_args.kwargs["where"] @@ -758,9 +772,7 @@ async def test_refresh_user_oauth_token_uses_client_secret_basic(monkeypatch): monkeypatch.setattr(db_mod, "get_async_httpx_client", lambda **kwargs: mock_client) monkeypatch.setattr(db_mod, "store_user_oauth_credential", AsyncMock()) - monkeypatch.setattr( - db_mod, "get_user_oauth_credential", AsyncMock(return_value={"access_token": "new-at"}) - ) + monkeypatch.setattr(db_mod, "get_user_oauth_credential", AsyncMock(return_value={"access_token": "new-at"})) result = await db_mod.refresh_user_oauth_token( prisma_client=MagicMock(), @@ -799,9 +811,7 @@ async def test_refresh_user_oauth_token_defaults_to_client_secret_post(monkeypat monkeypatch.setattr(db_mod, "get_async_httpx_client", lambda **kwargs: mock_client) monkeypatch.setattr(db_mod, "store_user_oauth_credential", AsyncMock()) - monkeypatch.setattr( - db_mod, "get_user_oauth_credential", AsyncMock(return_value={"access_token": "new-at"}) - ) + monkeypatch.setattr(db_mod, "get_user_oauth_credential", AsyncMock(return_value={"access_token": "new-at"})) await db_mod.refresh_user_oauth_token( prisma_client=MagicMock(), diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index e8fca9ac6ab..e11d78d07ae 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -3328,8 +3328,34 @@ class TestMCPServerManager: assert store.invalidations == [("alice", "srv-1")] @pytest.mark.asyncio - async def test_invalidate_user_oauth_token_cache_swallows_store_errors(self): - """A cache-drop failure must not fail the credential write that triggered it.""" + async def test_invalidate_user_oauth_token_cache_drops_legacy_cache_too(self, monkeypatch): + """A per-user token can be served from the legacy per-user token cache as well as the v2 + store; the shared invalidation must evict both, or the path not evicted keeps serving a + token minted for a replaced credential row until its TTL.""" + from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module + + class _Store: + async def fetch(self, user_id: str, server_id: str): + return None + + async def invalidate(self, user_id: str, server_id: str) -> None: + return None + + legacy_deletes: list[tuple[str, str]] = [] + monkeypatch.setattr( + manager_module.mcp_per_user_token_cache, + "delete", + AsyncMock(side_effect=lambda uid, sid: legacy_deletes.append((uid, sid))), + ) + manager = MCPServerManager(per_user_oauth_token_store=_Store()) + await manager.invalidate_user_oauth_token_cache("alice", "srv-1") + assert legacy_deletes == [("alice", "srv-1")] + + @pytest.mark.asyncio + async def test_invalidate_user_oauth_token_cache_swallows_store_errors(self, monkeypatch): + """A cache-drop failure must not fail the credential write that triggered it, and the + legacy cache must still be evicted after the v2 store drop fails.""" + from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module class _Store: async def fetch(self, user_id: str, server_id: str): @@ -3338,8 +3364,15 @@ class TestMCPServerManager: async def invalidate(self, user_id: str, server_id: str) -> None: raise RuntimeError("redis down") + legacy_deletes: list[tuple[str, str]] = [] + monkeypatch.setattr( + manager_module.mcp_per_user_token_cache, + "delete", + AsyncMock(side_effect=lambda uid, sid: legacy_deletes.append((uid, sid))), + ) manager = MCPServerManager(per_user_oauth_token_store=_Store()) await manager.invalidate_user_oauth_token_cache("alice", "srv-1") + assert legacy_deletes == [("alice", "srv-1")] @pytest.mark.asyncio async def test_resolve_oauth2_headers_no_user_id(self): diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index 7339420ed68..7093b7c650e 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -719,6 +719,56 @@ describe("CreateMCPServer", () => { expect(oauthHook.reset).not.toHaveBeenCalled(); }); + it("does not refetch the tool preview with a discarded token after invalidation", async () => { + // Regression: handleFormValuesChange used to publish the pre-reset antd snapshot into + // formValues after clearHeldOAuthToken, so useTestMCPConnection kept the discarded OAuth + // material (the DCR client minted for the old identity) and sent it on the next tool-preview + // request. + await setupOAuthInteractive(); + const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "https://a.example.com/mcp" } }); + }); + act(() => { + oauthHook.onTokenReceived?.({ access_token: "stale-tok" }, { clientId: "client-a", clientSecret: "secret-a" }); + }); + const nameInput = document.getElementById("server_name") as HTMLInputElement; + await act(async () => { + fireEvent.change(nameInput, { target: { value: "Sync_FormValues" } }); + }); + vi.mocked(networking.testMCPToolsListRequest).mockClear(); + + await selectAntOption("Authentication", "API Key"); + + await waitFor(() => expect(vi.mocked(networking.testMCPToolsListRequest)).toHaveBeenCalled()); + for (const call of vi.mocked(networking.testMCPToolsListRequest).mock.calls) { + expect(call[1]?.credentials?.client_id).not.toBe("client-a"); + expect(call[1]?.credentials?.client_secret).not.toBe("secret-a"); + expect(call[1]?.credentials?.access_token).not.toBe("stale-tok"); + } + }); + + it("keeps the held token on an http to sse switch with the same url", async () => { + // Same url means the same resource/audience (RFC 8707): the minted token is still valid, so a + // pure transport swap between the two MCP wire protocols must not force a re-authorize. + await setupOAuthInteractive(); + const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "https://a.example.com/mcp" } }); + }); + act(() => { + oauthHook.onTokenReceived?.({ access_token: "tok-a" }, { clientId: "client-a", clientSecret: "secret-a" }); + }); + oauthHook.reset.mockClear(); + + await selectAntOption("Transport Type", "Server-Sent Events (SSE)"); + + await waitFor(() => { + expect(screen.getByPlaceholderText("https://your-mcp-server.com")).toBeInTheDocument(); + }); + expect(oauthHook.reset).not.toHaveBeenCalled(); + }); + it("includes token_validation in payload when token_validation_json is filled with valid JSON", async () => { vi.mocked(networking.createMCPServer).mockResolvedValue({ server_id: "new-server-oauth", diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 17c7e7c0b9a..24b8e9eaafb 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -16,6 +16,8 @@ import { MCP_OAUTH2_FLOW_INTERACTIVE, isClientForwardedTokenMode, getOAuthAuthorizationIdentity, + CLEARED_ON_INVALIDATION, + isHeldOAuthTokenStale, } from "./types"; import OAuthFormFields from "./OAuthFormFields"; import TruePassthroughWarning from "./TruePassthroughWarning"; @@ -235,11 +237,9 @@ const CreateMCPServer: React.FC = ({ }); // Discard the held browser-authorized token and its tool preview when the authorization identity - // changes (or the modal closes). For oauth2 the fetched token + DCR client also live in - // form.credentials, and the discovered endpoints in authorization_url/token_url/registration_url, so - // those form fields are reset too; whatever the admin just changed (passed via changedValues) is + // changes (or the modal closes). The CLEARED_ON_INVALIDATION form fields (shared with the edit form + // via types.tsx) are reset too; whatever the admin just changed (passed via changedValues) is // re-applied so the invalidation never wipes their in-flight edit. - const CLEARED_ON_INVALIDATION = ["credentials", "authorization_url", "token_url", "registration_url"] as const; const clearHeldOAuthToken = (changedValues: Record = {}) => { setOauthAccessToken(null); clearTools(); @@ -588,21 +588,11 @@ const CreateMCPServer: React.FC = ({ ? { url: undefined, command: undefined, args: undefined, env: undefined } : { spec_path: undefined, command: undefined, args: undefined, env: undefined }; - const nextValues = - authorizedIdentity === undefined - ? transportValues - : { - ...transportValues, - credentials: undefined, - authorization_url: undefined, - token_url: undefined, - registration_url: undefined, - }; - - form.setFieldsValue(nextValues); - if (authorizedIdentity !== undefined) { + form.setFieldsValue(transportValues); + if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentity)) { clearHeldOAuthToken(); } + setFormValues(form.getFieldsValue(true)); }; // Generate options with existing groups and potential new group @@ -672,9 +662,13 @@ const CreateMCPServer: React.FC = ({ const handleFormValuesChange = (changedValues: Record, allValues: Record) => { // Any change to a mint-relevant field (url, auth_type, oauth_flow_type, client creds/scopes, or the // authorization/token/registration endpoints — see getOAuthAuthorizationIdentity) makes a held token - // stale, so discard it and force a fresh authorize. - if (authorizedIdentity !== undefined && getOAuthAuthorizationIdentity(allValues) !== authorizedIdentity) { + // stale, so discard it and force a fresh authorize. When that happens, formValues must be rebuilt + // from the form's post-reset state, not the pre-reset allValues snapshot: the snapshot still holds + // the discarded token in credentials, and useTestMCPConnection reads formValues for tool preview. + if (isHeldOAuthTokenStale(allValues, authorizedIdentity)) { clearHeldOAuthToken(changedValues); + setFormValues({ ...form.getFieldsValue(true), ...changedValues }); + return; } setFormValues(allValues); }; diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index d55f993b926..4f3d7b69b01 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -22,15 +22,22 @@ vi.mock("../molecules/notifications_manager", () => ({ const mockOauth: { tokenResponse: any; getTemporaryPayload: (() => Record | null) | null; -} = { tokenResponse: null, getTemporaryPayload: null }; + onTokenReceived: ((token: Record | null) => void) | null; + reset: ReturnType; +} = { tokenResponse: null, getTemporaryPayload: null, onTokenReceived: null, reset: vi.fn() }; vi.mock("@/hooks/useMcpOAuthFlow", () => ({ - useMcpOAuthFlow: (opts: { getTemporaryPayload?: () => Record | null }) => { + useMcpOAuthFlow: (opts: { + getTemporaryPayload?: () => Record | null; + onTokenReceived?: (token: Record | null) => void; + }) => { mockOauth.getTemporaryPayload = opts?.getTemporaryPayload ?? null; + mockOauth.onTokenReceived = opts?.onTokenReceived ?? null; return { startOAuthFlow: vi.fn(), status: "idle", error: null, tokenResponse: mockOauth.tokenResponse, + reset: mockOauth.reset, }; }, })); @@ -92,10 +99,12 @@ vi.mock("./mcp_tool_configuration", () => ({ const mockGetToken = vi.fn(); const mockIsTokenValid = vi.fn(); const mockSetToken = vi.fn(); +const mockRemoveToken = vi.fn(); vi.mock("@/utils/mcpTokenStore", () => ({ getToken: (...args: any[]) => mockGetToken(...args), isTokenValid: (...args: any[]) => mockIsTokenValid(...args), setToken: (...args: any[]) => mockSetToken(...args), + removeToken: (...args: unknown[]) => mockRemoveToken(...args), })); // ── fixtures ────────────────────────────────────────────────────────────────── @@ -451,6 +460,74 @@ describe("MCPServerEdit (auth type switch)", () => { }); }); +describe("MCPServerEdit OAuth token invalidation", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + const renderOAuthEdit = () => + render( + , + ); + + it("invalidates a session-authorized token when the transport switches to stdio", async () => { + // Switching to stdio clears url/auth_type via programmatic form.setFieldsValue, which antd does + // not report through onValuesChange; the explicit recheck in handleTransportChange must catch it. + // Regression: the token used to survive this switch (sessionStorage + hook state kept the old + // token minted for the http url). + renderOAuthEdit(); + + act(() => { + mockOauth.onTokenReceived?.({ access_token: "tok-1" }); + }); + mockOauth.reset.mockClear(); + + await selectAntOption("Transport Type", "Standard Input/Output (stdio)"); + + await waitFor(() => expect(mockOauth.reset).toHaveBeenCalled()); + expect(mockRemoveToken).toHaveBeenCalledWith("oauth_server_1", undefined); + }); + + it("invalidates a session-authorized token when the server URL changes", async () => { + renderOAuthEdit(); + + act(() => { + mockOauth.onTokenReceived?.({ access_token: "tok-1" }); + }); + mockOauth.reset.mockClear(); + + const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "https://other.example.com/mcp" } }); + }); + + await waitFor(() => expect(mockOauth.reset).toHaveBeenCalled()); + expect(mockRemoveToken).toHaveBeenCalledWith("oauth_server_1", undefined); + }); + + it("keeps a session-authorized token on an http to sse switch with the same url", async () => { + // Same url means the same resource/audience (RFC 8707): the minted token is still valid, so a + // pure transport swap between the two MCP wire protocols must not force a re-authorize. + renderOAuthEdit(); + + act(() => { + mockOauth.onTokenReceived?.({ access_token: "tok-1" }); + }); + mockOauth.reset.mockClear(); + + await selectAntOption("Transport Type", "Server-Sent Events (SSE)"); + + expect(mockOauth.reset).not.toHaveBeenCalled(); + expect(mockRemoveToken).not.toHaveBeenCalled(); + }); +}); + describe("MCPServerEdit (tool allowlist)", () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 55adcf2bb59..de3528ea6a9 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -6,6 +6,8 @@ import { AUTH_TYPE, isClientForwardedTokenMode, getOAuthAuthorizationIdentity, + CLEARED_ON_INVALIDATION, + isHeldOAuthTokenStale, OAUTH_FLOW, MCP_OAUTH2_FLOW_M2M, MCP_OAUTH2_FLOW_INTERACTIVE, @@ -392,11 +394,12 @@ const MCPServerEdit: React.FC = ({ // identity it was minted against (url, auth_type, oauth_flow_type, client creds/scopes, or the // authorization/token/registration endpoints — see getOAuthAuthorizationIdentity). Discards the hook // token (resetOAuthFlow, which re-runs fetchTools to prompt a fresh authorize), the sessionStorage - // token (removeToken, browser-held modes), and the fetched token/DCR client in form.credentials + the - // discovered endpoint fields; the admin's in-flight edit is re-applied so it is never wiped. Only fires - // when a token was actually authorized here (ref set), so a token already valid for the saved server on - // mount is left untouched. Driven from onValuesChange (user input only), never programmatic resets. - const CLEARED_ON_INVALIDATION = ["credentials", "authorization_url", "token_url", "registration_url"] as const; + // token (removeToken, browser-held modes), and the fetched token/DCR client in the shared + // CLEARED_ON_INVALIDATION form fields; the admin's in-flight edit is re-applied so it is never wiped. + // Only fires when a token was actually authorized here (ref set), so a token already valid for the + // saved server on mount is left untouched. Driven from onValuesChange for user input, plus an explicit + // recheck after programmatic setFieldsValue paths (handleTransportChange), which antd does not report + // through onValuesChange. const clearHeldOAuthToken = (changedValues: Record = {}) => { authorizedIdentityRef.current = undefined; if (mcpServer.server_id) { @@ -413,10 +416,7 @@ const MCPServerEdit: React.FC = ({ }; const handleFormValuesChange = (changedValues: Record) => { - if ( - authorizedIdentityRef.current !== undefined && - getOAuthAuthorizationIdentity(form.getFieldsValue(true)) !== authorizedIdentityRef.current - ) { + if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentityRef.current)) { clearHeldOAuthToken(changedValues); } }; @@ -539,6 +539,9 @@ const MCPServerEdit: React.FC = ({ stdio_config: undefined, }); } + if (isHeldOAuthTokenStale(form.getFieldsValue(true), authorizedIdentityRef.current)) { + clearHeldOAuthToken(); + } }; const handleSave = async (values: Record) => { diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index 7386fc98bc2..e894cf4b8dc 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -84,6 +84,21 @@ export const getOAuthAuthorizationIdentity = (values: Record): return JSON.stringify(identity); }; +// The form fields wiped when a held OAuth token is invalidated: the fetched token + DCR client live in +// `credentials`, and the three endpoint fields were discovered by the authorize flow, so all of them are +// stale together with the token. Shared by the create and edit forms so what gets wiped cannot drift. +export const CLEARED_ON_INVALIDATION = ["credentials", "authorization_url", "token_url", "registration_url"] as const; + +// True when a token was authorized in this session (authorizedIdentity recorded at mint time) and the +// form's current identity no longer matches it. Every invalidation decision in both forms goes through +// this single check: onValuesChange for user edits, and an explicit recheck after any programmatic +// form.setFieldsValue (antd does not fire onValuesChange for those), so a missed event path cannot let a +// stale token survive. +export const isHeldOAuthTokenStale = ( + values: Record, + authorizedIdentity: string | undefined, +): boolean => authorizedIdentity !== undefined && getOAuthAuthorizationIdentity(values) !== authorizedIdentity; + // Backend value of `oauth2_flow` that marks a machine-to-machine server. Distinct // from the UI-local OAUTH_FLOW.M2M ("m2m"); this is what the API actually returns. export const MCP_OAUTH2_FLOW_M2M = "client_credentials"; From 42388c3d689807f5e94de9311c40a09448bb488f Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 12:52:18 -0700 Subject: [PATCH 080/101] refactor(mcp): align the invalidation code with the v2 DI and typing discipline The purge takes an injectable invalidate_token_cache callable defaulting to the manager's shared invalidation, and MCPServerManager takes an injectable per_user_token_cache alongside the existing per_user_oauth_token_store, so tests inject fakes instead of monkeypatching the global manager and the module-level cache. The new identity helpers drop Any for object throughout --- litellm/proxy/_experimental/mcp_server/db.py | 32 +++++++++----- .../mcp_server/mcp_server_manager.py | 5 ++- .../mcp_server/test_db_credentials.py | 28 +++++-------- .../mcp_server/test_mcp_server_manager.py | 42 ++++++++++--------- 4 files changed, 57 insertions(+), 50 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 135a9e055d6..96b28afc093 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -3,7 +3,7 @@ import binascii import hashlib import json from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Set, Union, cast +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Iterable, List, Optional, Set, Union, cast from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -1070,7 +1070,7 @@ async def list_user_oauth_credentials( return results -def _decrypted_credential_field(creds: Dict[str, Any], field: str) -> Any: +def _decrypted_credential_field(creds: Dict[str, object], field: str) -> object: """Return one credential field decrypted with the global salt key; non-string and legacy plaintext values come back unchanged (decrypt_value_helper returns the original on failure).""" value = creds.get(field) @@ -1084,7 +1084,7 @@ def _decrypted_credential_field(creds: Dict[str, Any], field: str) -> Any: ) -def mcp_oauth_token_identity(server: Any) -> tuple[Any, ...]: +def mcp_oauth_token_identity(server: object) -> tuple[object, ...]: """The upstream-OAuth-token-determining fields of an MCP server: the resource/audience (url, or spec_path for OpenAPI servers), the OAuth mode/grant (auth_type, oauth2_flow), the authorization-server endpoints, and the OAuth client + scopes. Mirrors the dashboard's @@ -1098,12 +1098,12 @@ def mcp_oauth_token_identity(server: Any) -> tuple[Any, ...]: creds = getattr(server, "credentials", None) if isinstance(creds, str): try: - parsed: Any = json.loads(creds) + parsed: object = json.loads(creds) except ValueError: parsed = None else: parsed = creds - creds_dict: Dict[str, Any] = parsed if isinstance(parsed, dict) else {} + creds_dict: Dict[str, object] = parsed if isinstance(parsed, dict) else {} return ( getattr(server, "url", None), getattr(server, "spec_path", None), @@ -1118,25 +1118,35 @@ def mcp_oauth_token_identity(server: Any) -> tuple[Any, ...]: ) -async def purge_user_oauth_credentials_for_server(prisma_client: PrismaClient, server_id: str) -> int: +async def purge_user_oauth_credentials_for_server( + prisma_client: PrismaClient, + server_id: str, + invalidate_token_cache: Optional[Callable[[str, str], Awaitable[None]]] = None, +) -> int: """Delete every stored per-user OAuth credential for a server and invalidate each user's cached token everywhere it can be served from (the legacy per-user token cache and the v2 per-user OAuth token store), so no user keeps a token minted for a superseded configuration. Called when a server update changes a mint-relevant field (see mcp_oauth_token_identity). Returns the number of rows removed. A row inserted between the find and the delete is removed from the DB but cannot be evicted from the caches (its user_id was never seen); that case is detected, logged, and bounded - by the cache TTL.""" + by the cache TTL. + + invalidate_token_cache is injectable for tests; it defaults to the manager's shared + invalidate_user_oauth_token_cache, the single invalidation point for per-user tokens.""" repo = MCPUserCredentialsRepository(prisma_client) rows = await repo.table.find_many(where={"server_id": server_id}) if not rows: return 0 deleted_count = await repo.table.delete_many(where={"server_id": server_id}) - from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( - global_mcp_server_manager, - ) + if invalidate_token_cache is None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + invalidate_token_cache = global_mcp_server_manager.invalidate_user_oauth_token_cache for row in rows: - await global_mcp_server_manager.invalidate_user_oauth_token_cache(row.user_id, server_id) + await invalidate_token_cache(row.user_id, server_id) if deleted_count != len(rows): verbose_proxy_logger.warning( "MCP server %s: purge removed %d credential row(s) but %d were enumerated; " diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index cc4a9e63105..41a87a17d58 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -58,6 +58,7 @@ from litellm.proxy._experimental.mcp_server.sampling_handler import ( MCP_SAMPLING_AVAILABLE, ) from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( + MCPPerUserTokenCache, mcp_per_user_token_cache, resolve_mcp_auth, ) @@ -802,10 +803,12 @@ class MCPServerManager: self, cred_provider: Optional[UpstreamCredentialProvider] = None, per_user_oauth_token_store: Optional[InvalidatableOAuthTokenStore] = None, + per_user_token_cache: Optional[MCPPerUserTokenCache] = None, ): self._per_user_oauth_token_store = per_user_oauth_token_store or LazyPerUserOAuthTokenStore( self.get_mcp_server_by_id ) + self._per_user_token_cache = per_user_token_cache or mcp_per_user_token_cache self._cred_provider = cred_provider or UpstreamCredentialProvider( oauth_token_store=self._per_user_oauth_token_store, token_exchanger=build_token_exchanger(), @@ -4070,7 +4073,7 @@ class MCPServerManager: verbose_logger.warning( "Failed to invalidate cached MCP OAuth token for user=%s server=%s: %s", user_id, server_id, exc ) - await mcp_per_user_token_cache.delete(user_id, server_id) + await self._per_user_token_cache.delete(user_id, server_id) async def _resolve_oauth2_headers_for_tool_call( self, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 51641991ef9..1615d81fae9 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -149,11 +149,11 @@ def test_mcp_oauth_token_identity_detects_change_under_encryption(): @pytest.mark.asyncio -async def test_purge_user_oauth_credentials_for_server_invalidates_every_store(monkeypatch): - """The purge must route each (user, server) through the manager's shared invalidation, which is - the single point covering both the legacy per-user token cache and the v2 per-user OAuth token - store; evicting only one cache lets the other keep serving a token minted for the old config.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager +async def test_purge_user_oauth_credentials_for_server_invalidates_every_store(): + """The purge must route each (user, server) through the injected invalidator (defaulting to the + manager's shared invalidation, the single point covering both the legacy per-user token cache and + the v2 per-user OAuth token store); evicting only one cache lets the other keep serving a token + minted for the old config.""" from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server r1 = MagicMock(user_id="alice", server_id="srv-1") @@ -163,13 +163,11 @@ async def test_purge_user_oauth_credentials_for_server_invalidates_every_store(m prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) invalidations = [] - monkeypatch.setattr( - mcp_server_manager.global_mcp_server_manager, - "invalidate_user_oauth_token_cache", - AsyncMock(side_effect=lambda uid, sid: invalidations.append((uid, sid))), - ) - purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") + async def record_invalidation(user_id: str, server_id: str) -> None: + invalidations.append((user_id, server_id)) + + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1", invalidate_token_cache=record_invalidation) assert purged == 2 prisma.db.litellm_mcpusercredentials.delete_many.assert_awaited_once() @@ -179,7 +177,6 @@ async def test_purge_user_oauth_credentials_for_server_invalidates_every_store(m @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_logs_raced_rows(monkeypatch): from litellm.proxy._experimental.mcp_server import db as db_module - from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() @@ -187,15 +184,10 @@ async def test_purge_user_oauth_credentials_for_server_logs_raced_rows(monkeypat return_value=[MagicMock(user_id="alice", server_id="srv-1")] ) prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) - monkeypatch.setattr( - mcp_server_manager.global_mcp_server_manager, - "invalidate_user_oauth_token_cache", - AsyncMock(), - ) warning = MagicMock() monkeypatch.setattr(db_module.verbose_proxy_logger, "warning", warning) - purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1", invalidate_token_cache=AsyncMock()) assert purged == 2 warning.assert_called_once() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index e11d78d07ae..97fdd186dda 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -3328,11 +3328,10 @@ class TestMCPServerManager: assert store.invalidations == [("alice", "srv-1")] @pytest.mark.asyncio - async def test_invalidate_user_oauth_token_cache_drops_legacy_cache_too(self, monkeypatch): + async def test_invalidate_user_oauth_token_cache_drops_legacy_cache_too(self): """A per-user token can be served from the legacy per-user token cache as well as the v2 store; the shared invalidation must evict both, or the path not evicted keeps serving a token minted for a replaced credential row until its TTL.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module class _Store: async def fetch(self, user_id: str, server_id: str): @@ -3341,21 +3340,22 @@ class TestMCPServerManager: async def invalidate(self, user_id: str, server_id: str) -> None: return None - legacy_deletes: list[tuple[str, str]] = [] - monkeypatch.setattr( - manager_module.mcp_per_user_token_cache, - "delete", - AsyncMock(side_effect=lambda uid, sid: legacy_deletes.append((uid, sid))), - ) - manager = MCPServerManager(per_user_oauth_token_store=_Store()) + class _LegacyCache: + def __init__(self) -> None: + self.deletes: list[tuple[str, str]] = [] + + async def delete(self, user_id: str, server_id: str) -> None: + self.deletes.append((user_id, server_id)) + + legacy_cache = _LegacyCache() + manager = MCPServerManager(per_user_oauth_token_store=_Store(), per_user_token_cache=legacy_cache) await manager.invalidate_user_oauth_token_cache("alice", "srv-1") - assert legacy_deletes == [("alice", "srv-1")] + assert legacy_cache.deletes == [("alice", "srv-1")] @pytest.mark.asyncio - async def test_invalidate_user_oauth_token_cache_swallows_store_errors(self, monkeypatch): + async def test_invalidate_user_oauth_token_cache_swallows_store_errors(self): """A cache-drop failure must not fail the credential write that triggered it, and the legacy cache must still be evicted after the v2 store drop fails.""" - from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module class _Store: async def fetch(self, user_id: str, server_id: str): @@ -3364,15 +3364,17 @@ class TestMCPServerManager: async def invalidate(self, user_id: str, server_id: str) -> None: raise RuntimeError("redis down") - legacy_deletes: list[tuple[str, str]] = [] - monkeypatch.setattr( - manager_module.mcp_per_user_token_cache, - "delete", - AsyncMock(side_effect=lambda uid, sid: legacy_deletes.append((uid, sid))), - ) - manager = MCPServerManager(per_user_oauth_token_store=_Store()) + class _LegacyCache: + def __init__(self) -> None: + self.deletes: list[tuple[str, str]] = [] + + async def delete(self, user_id: str, server_id: str) -> None: + self.deletes.append((user_id, server_id)) + + legacy_cache = _LegacyCache() + manager = MCPServerManager(per_user_oauth_token_store=_Store(), per_user_token_cache=legacy_cache) await manager.invalidate_user_oauth_token_cache("alice", "srv-1") - assert legacy_deletes == [("alice", "srv-1")] + assert legacy_cache.deletes == [("alice", "srv-1")] @pytest.mark.asyncio async def test_resolve_oauth2_headers_no_user_id(self): From c75184bec9b6c5337e5c810e59d34635eb340ecf Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 13:10:24 -0700 Subject: [PATCH 081/101] fix(mcp): make the pre-update identity snapshot advisory so a read failure cannot fail the edit The snapshot read only feeds the stale-token purge decision; leaving it unguarded meant a failed read would 500 an edit whose update would have succeeded, and it broke test_edit_mcp_server_redacts_credentials, whose mocked prisma is not awaitable on the un-patched get_mcp_server path. A failure now logs and skips the purge, consistent with the purge half already being best-effort. Adds the first endpoint-level coverage of the edit purge wiring: purge on a mint-relevant change, no purge when the identity is unchanged, and edit success with purge skipped when the snapshot read raises --- .../mcp_management_endpoints.py | 14 +++- .../test_mcp_management_endpoints.py | 76 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 8f6779b17b9..907a17d76d9 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -2320,8 +2320,18 @@ if MCP_AVAILABLE: }, ) - # Snapshot the pre-update identity so we can detect a mint-relevant change below. - old_server_record = await get_mcp_server(prisma_client, payload.server_id) + # Snapshot the pre-update identity so we can detect a mint-relevant change below. The read is + # advisory (it only feeds the stale-token purge decision), so a failure skips the purge with a + # warning instead of failing the edit, whose primary job is the update itself. + try: + old_server_record = await get_mcp_server(prisma_client, payload.server_id) + except Exception as exc: # noqa: BLE001 - advisory read; invalidation is best-effort end-to-end + verbose_logger.warning( + "MCP server %s: could not snapshot the pre-update record; skipping the stale-token check: %s", + payload.server_id, + exc, + ) + old_server_record = None # try to update the mcp server mcp_server_record_updated = await update_mcp_server( diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index 86bbce36de3..a8a8ee0fbde 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -5134,3 +5134,79 @@ def test_stamp_oauth2_flow_ignores_non_oauth2(): payload = _oauth2_create_payload(auth_type="none") mgmt_endpoints.stamp_omitted_oauth2_flow(payload) assert payload.oauth2_flow is None + + +async def _run_edit(old_record, updated_record): + from litellm.proxy.management_endpoints.mcp_management_endpoints import edit_mcp_server + + server_id = updated_record.server_id + with ( + patch("litellm.proxy.management_endpoints.mcp_management_endpoints.MCP_AVAILABLE", True), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw", + return_value=MagicMock(), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + AsyncMock(side_effect=old_record) + if isinstance(old_record, Exception) + else AsyncMock(return_value=old_record), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.update_mcp_server", + AsyncMock(return_value=updated_record), + ), + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.validate_and_normalize_mcp_server_payload", + autospec=True, + ), + patch("litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager") as mock_manager, + patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.purge_user_oauth_credentials_for_server", + AsyncMock(return_value=1), + ) as mock_purge, + ): + mock_manager.update_server = AsyncMock() + mock_manager.reload_servers_from_database = AsyncMock() + payload = UpdateMCPServerRequest(server_id=server_id, alias=updated_record.alias, url=updated_record.url) + user_auth = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN) + result = await edit_mcp_server(payload=payload, user_api_key_dict=user_auth) + return result, mock_purge + + +@pytest.mark.asyncio +async def test_edit_mcp_server_purges_user_tokens_on_mint_relevant_change(): + server_id = str(uuid.uuid4()) + old = generate_mock_mcp_server_db_record(server_id=server_id, url="https://old.example.com/mcp") + updated = generate_mock_mcp_server_db_record(server_id=server_id, url="https://new.example.com/mcp") + + result, mock_purge = await _run_edit(old, updated) + + assert result.server_id == server_id + mock_purge.assert_awaited_once() + assert mock_purge.await_args.args[1] == server_id + + +@pytest.mark.asyncio +async def test_edit_mcp_server_skips_purge_when_identity_unchanged(): + server_id = str(uuid.uuid4()) + old = generate_mock_mcp_server_db_record(server_id=server_id, alias="Before") + updated = generate_mock_mcp_server_db_record(server_id=server_id, alias="After") + + result, mock_purge = await _run_edit(old, updated) + + assert result.server_id == server_id + mock_purge.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_edit_mcp_server_snapshot_failure_skips_purge_but_edit_succeeds(): + """The pre-update snapshot read is advisory (it only feeds the purge decision); a read failure + must skip the stale-token check with a warning, never fail the edit itself.""" + server_id = str(uuid.uuid4()) + updated = generate_mock_mcp_server_db_record(server_id=server_id, url="https://new.example.com/mcp") + + result, mock_purge = await _run_edit(RuntimeError("db read failed"), updated) + + assert result.server_id == server_id + mock_purge.assert_not_awaited() From e720b5e25a7dfa3365fd5e32b287b906daaa4216 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 13:53:59 -0700 Subject: [PATCH 082/101] test(ui): drop the vacuous access_token assertion from the preview invalidation test The staged access token never reaches formValues (it is not a registered form field), so the assertion could not fail; the DCR client pair is the leak the test actually pins, proven by the mutation run --- .../src/components/mcp_tools/create_mcp_server.test.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index 7093b7c650e..658116ede1f 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -744,7 +744,6 @@ describe("CreateMCPServer", () => { for (const call of vi.mocked(networking.testMCPToolsListRequest).mock.calls) { expect(call[1]?.credentials?.client_id).not.toBe("client-a"); expect(call[1]?.credentials?.client_secret).not.toBe("secret-a"); - expect(call[1]?.credentials?.access_token).not.toBe("stale-tok"); } }); From aa351311c0c7edb7f3d52df7a11a9c4c49af0cd9 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 14:33:39 -0700 Subject: [PATCH 083/101] fix(mcp): spare BYOK rows when purging stale OAuth tokens and invalidate caches on server delete LiteLLM_MCPUserCredentials stores BYOK API keys in the same column as per-user OAuth tokens, so the purge on a mint-relevant config change now deletes only rows whose payload decodes as an OAuth2 credential, each by its (user_id, server_id) pair, instead of every row for the server. An api_key server whose url changes purges nothing. delete_mcp_server now also invalidates each enumerated user's cached token so a re-created server reusing the id cannot serve tokens minted for the deleted one, and both cache drops are best-effort --- litellm/proxy/_experimental/mcp_server/db.py | 64 ++++++-- .../mcp_server/mcp_server_manager.py | 7 +- .../mcp_server/test_db_credentials.py | 143 ++++++++++++++++-- .../mcp_server/test_mcp_server.py | 1 + .../mcp_server/test_mcp_server_manager.py | 19 +++ .../test_mcp_management_endpoints.py | 18 ++- 6 files changed, 222 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 96b28afc093..c6b7620b649 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -558,7 +558,11 @@ async def delete_mcp_server_from_virtualkey(): pass -async def delete_mcp_server(prisma_client: PrismaClient, server_id: str) -> Optional[LiteLLM_MCPServerTable]: +async def delete_mcp_server( + prisma_client: PrismaClient, + server_id: str, + invalidate_token_cache: Optional[Callable[[str, str], Awaitable[None]]] = None, +) -> Optional[LiteLLM_MCPServerTable]: """ Delete the mcp server from the db by server_id @@ -569,6 +573,12 @@ async def delete_mcp_server(prisma_client: PrismaClient, server_id: str) -> Opti caller-visible error. Each table is cleaned independently so a failure on one still attempts the other. + Each enumerated credential row's user also gets their cached per-user token + invalidated (legacy cache + v2 store, via invalidate_token_cache, defaulting + to the manager's shared invalidation): the caches are keyed by + (user_id, server_id), so without this a re-created server reusing the same + server_id would serve tokens minted for the deleted server until TTL. + Returns the deleted mcp server record if it exists, otherwise None """ deleted_server = await MCPServerRepository(prisma_client).table.delete( @@ -577,6 +587,18 @@ async def delete_mcp_server(prisma_client: PrismaClient, server_id: str) -> Opti }, ) if deleted_server is not None: + credential_user_ids: List[str] = [] + try: + credential_rows = await prisma_client.db.litellm_mcpusercredentials.find_many( + where={"server_id": server_id} + ) + credential_user_ids = [row.user_id for row in credential_rows] + except Exception as e: # noqa: BLE001 - enumeration is best-effort; cached tokens expire by TTL + verbose_proxy_logger.warning( + "MCP server %s deleted but per-user credential enumeration failed; cached tokens expire by TTL: %s", + server_id, + e, + ) for model, label in ( (prisma_client.db.litellm_mcpusercredentials, "credential"), (prisma_client.db.litellm_mcpuserenvvars, "env var"), @@ -591,6 +613,15 @@ async def delete_mcp_server(prisma_client: PrismaClient, server_id: str) -> Opti label, e, ) + if credential_user_ids: + if invalidate_token_cache is None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + invalidate_token_cache = global_mcp_server_manager.invalidate_user_oauth_token_cache + for user_id in credential_user_ids: + await invalidate_token_cache(user_id, server_id) return deleted_server @@ -1123,21 +1154,30 @@ async def purge_user_oauth_credentials_for_server( server_id: str, invalidate_token_cache: Optional[Callable[[str, str], Awaitable[None]]] = None, ) -> int: - """Delete every stored per-user OAuth credential for a server and invalidate each user's cached + """Delete every stored per-user OAuth token for a server and invalidate each user's cached token everywhere it can be served from (the legacy per-user token cache and the v2 per-user OAuth token store), so no user keeps a token minted for a superseded configuration. Called when a server update changes a mint-relevant field (see mcp_oauth_token_identity). Returns the number of rows - removed. A row inserted between the find and the delete is removed from the DB but cannot be - evicted from the caches (its user_id was never seen); that case is detected, logged, and bounded - by the cache TTL. + removed. + + LiteLLM_MCPUserCredentials also stores BYOK API keys in the same column; only rows whose payload + decodes as an OAuth2 credential (see _decode_oauth_payload) are deleted, because a config change + only invalidates minted tokens, never a user's own stored key. Rows are therefore deleted per + (user_id, server_id) pair rather than by a blanket server_id filter. An OAuth row inserted while + the purge runs for a user not yet enumerated survives; a re-auth completing in the window for an + already-enumerated user is deleted along with the stale row (the pair delete cannot tell them + apart), which costs that user one extra re-auth and nothing else. invalidate_token_cache is injectable for tests; it defaults to the manager's shared invalidate_user_oauth_token_cache, the single invalidation point for per-user tokens.""" repo = MCPUserCredentialsRepository(prisma_client) rows = await repo.table.find_many(where={"server_id": server_id}) - if not rows: + oauth_rows = [row for row in rows if _decode_oauth_payload(row.credential_b64) is not None] + if not oauth_rows: return 0 - deleted_count = await repo.table.delete_many(where={"server_id": server_id}) + deleted_count = sum( + [await repo.table.delete_many(where={"user_id": row.user_id, "server_id": server_id}) for row in oauth_rows] + ) if invalidate_token_cache is None: from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, @@ -1145,15 +1185,15 @@ async def purge_user_oauth_credentials_for_server( invalidate_token_cache = global_mcp_server_manager.invalidate_user_oauth_token_cache - for row in rows: + for row in oauth_rows: await invalidate_token_cache(row.user_id, server_id) - if deleted_count != len(rows): + if deleted_count != len(oauth_rows): verbose_proxy_logger.warning( - "MCP server %s: purge removed %d credential row(s) but %d were enumerated; " - "row(s) raced in during the purge and their cached tokens will expire by TTL", + "MCP server %s: purge removed %d OAuth credential row(s) but %d were enumerated; " + "row(s) were deleted concurrently during the purge", server_id, deleted_count, - len(rows), + len(oauth_rows), ) return deleted_count diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 41a87a17d58..8dd9949e17b 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -4073,7 +4073,12 @@ class MCPServerManager: verbose_logger.warning( "Failed to invalidate cached MCP OAuth token for user=%s server=%s: %s", user_id, server_id, exc ) - await self._per_user_token_cache.delete(user_id, server_id) + try: + await self._per_user_token_cache.delete(user_id, server_id) + except Exception as exc: # noqa: BLE001 - cache drop is best-effort; TTL is the backstop + verbose_logger.warning( + "Failed to drop legacy cached MCP OAuth token for user=%s server=%s: %s", user_id, server_id, exc + ) async def _resolve_oauth2_headers_for_tool_call( self, diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 1615d81fae9..0bdcffe1530 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -148,19 +148,30 @@ def test_mcp_oauth_token_identity_detects_change_under_encryption(): assert mcp_oauth_token_identity(unchanged) != mcp_oauth_token_identity(changed) +def _oauth_row(user_id: str, server_id: str = "srv-1"): + """A stored per-user OAuth token row (payload tagged type=oauth2, legacy plain-base64 encoding).""" + row = _legacy_row(json.dumps({"type": "oauth2", "access_token": "tok-" + user_id})) + row.user_id = user_id + row.server_id = server_id + return row + + +def _byok_row(user_id: str, server_id: str = "srv-1"): + """A stored BYOK API key row: the same column, but the payload is a plain string, not OAuth JSON.""" + row = _legacy_row("sk-byok-" + user_id) + row.user_id = user_id + row.server_id = server_id + return row + + @pytest.mark.asyncio -async def test_purge_user_oauth_credentials_for_server_invalidates_every_store(): - """The purge must route each (user, server) through the injected invalidator (defaulting to the - manager's shared invalidation, the single point covering both the legacy per-user token cache and - the v2 per-user OAuth token store); evicting only one cache lets the other keep serving a token - minted for the old config.""" +async def test_purge_user_oauth_credentials_for_server_invalidates_each_user(): + """The purge must route each (user, server) row through the invalidator exactly once.""" from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server - r1 = MagicMock(user_id="alice", server_id="srv-1") - r2 = MagicMock(user_id="bob", server_id="srv-1") prisma = MagicMock() - prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[r1, r2]) - prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice"), _oauth_row("bob")]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=1) invalidations = [] @@ -170,29 +181,131 @@ async def test_purge_user_oauth_credentials_for_server_invalidates_every_store() purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1", invalidate_token_cache=record_invalidation) assert purged == 2 - prisma.db.litellm_mcpusercredentials.delete_many.assert_awaited_once() + assert prisma.db.litellm_mcpusercredentials.delete_many.await_count == 2 assert set(invalidations) == {("alice", "srv-1"), ("bob", "srv-1")} +@pytest.mark.asyncio +async def test_purge_user_oauth_credentials_for_server_spares_byok_rows(): + """Regression: the purge used to delete_many on server_id alone, wiping BYOK API keys that share + the LiteLLM_MCPUserCredentials table. Only rows holding an OAuth2 payload may be deleted, each by + its (user_id, server_id) pair, and only their users' token caches invalidated.""" + from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + + prisma = MagicMock() + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_byok_row("carol"), _oauth_row("alice")]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=1) + + invalidations = [] + + async def record_invalidation(user_id: str, server_id: str) -> None: + invalidations.append((user_id, server_id)) + + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1", invalidate_token_cache=record_invalidation) + + assert purged == 1 + prisma.db.litellm_mcpusercredentials.delete_many.assert_awaited_once_with( + where={"user_id": "alice", "server_id": "srv-1"} + ) + assert invalidations == [("alice", "srv-1")] + + +@pytest.mark.asyncio +async def test_purge_user_oauth_credentials_for_server_all_byok_is_noop(): + """An api_key (BYOK-only) server whose identity tuple changes (e.g. its url) must purge nothing.""" + from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + + prisma = MagicMock() + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_byok_row("carol"), _byok_row("dave")]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock() + + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") + + assert purged == 0 + prisma.db.litellm_mcpusercredentials.delete_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_purge_user_oauth_credentials_for_server_defaults_to_manager_invalidator(monkeypatch): + """When no invalidator is injected, the purge must resolve to the manager's shared + invalidate_user_oauth_token_cache, the single point covering both the legacy per-user token cache + and the v2 per-user OAuth token store; a wrong or no-op default silently leaves every cache + serving tokens minted for the superseded config.""" + from litellm.proxy._experimental.mcp_server import mcp_server_manager + from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server + + prisma = MagicMock() + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice")]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=1) + + shared_invalidator = AsyncMock() + monkeypatch.setattr( + mcp_server_manager.global_mcp_server_manager, + "invalidate_user_oauth_token_cache", + shared_invalidator, + ) + + purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1") + + assert purged == 1 + shared_invalidator.assert_awaited_once_with("alice", "srv-1") + + @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_logs_raced_rows(monkeypatch): from litellm.proxy._experimental.mcp_server import db as db_module from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() - prisma.db.litellm_mcpusercredentials.find_many = AsyncMock( - return_value=[MagicMock(user_id="alice", server_id="srv-1")] - ) - prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice")]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=0) warning = MagicMock() monkeypatch.setattr(db_module.verbose_proxy_logger, "warning", warning) purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1", invalidate_token_cache=AsyncMock()) - assert purged == 2 + assert purged == 0 warning.assert_called_once() +@pytest.mark.asyncio +async def test_delete_mcp_server_invalidates_cached_tokens_for_enumerated_users(): + """Deleting a server must invalidate each enumerated user's cached per-user token: the caches are + keyed by (user_id, server_id), so a re-created server reusing the same server_id would otherwise + serve tokens minted for the deleted server until TTL.""" + from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + + prisma = MagicMock() + prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=MagicMock(server_id="srv-1")) + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice"), _byok_row("bob")]) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) + prisma.db.litellm_mcpuserenvvars.delete_many = AsyncMock(return_value=0) + + invalidations = [] + + async def record_invalidation(user_id: str, server_id: str) -> None: + invalidations.append((user_id, server_id)) + + deleted = await delete_mcp_server(prisma, "srv-1", invalidate_token_cache=record_invalidation) + + assert deleted is not None + assert set(invalidations) == {("alice", "srv-1"), ("bob", "srv-1")} + + +@pytest.mark.asyncio +async def test_delete_mcp_server_returns_none_without_cleanup_when_server_missing(): + from litellm.proxy._experimental.mcp_server.db import delete_mcp_server + + prisma = MagicMock() + prisma.db.litellm_mcpservertable.delete = AsyncMock(return_value=None) + prisma.db.litellm_mcpusercredentials.find_many = AsyncMock() + + deleted = await delete_mcp_server(prisma, "srv-1", invalidate_token_cache=AsyncMock()) + + assert deleted is None + prisma.db.litellm_mcpusercredentials.find_many.assert_not_awaited() + + @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_noop_when_empty(): from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index bba0eb31cfb..7d25e0ba493 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -6869,6 +6869,7 @@ async def test_call_tool_with_legacy_db_m2m_server_resolves_oauth2_flow(): (None, None), ("", None), ("not a url", None), + ("http://[::1", None), ], ) def test_redact_mcp_resource_url_strips_credentials(url, expected): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 97fdd186dda..a18e1ac2c44 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -3376,6 +3376,25 @@ class TestMCPServerManager: await manager.invalidate_user_oauth_token_cache("alice", "srv-1") assert legacy_cache.deletes == [("alice", "srv-1")] + @pytest.mark.asyncio + async def test_invalidate_user_oauth_token_cache_swallows_legacy_cache_errors(self): + """The legacy cache drop is best-effort like the v2 drop: a failure must be logged, never + raised into the credential write that triggered the invalidation.""" + + class _Store: + async def fetch(self, user_id: str, server_id: str): + return None + + async def invalidate(self, user_id: str, server_id: str) -> None: + return None + + class _RaisingLegacyCache: + async def delete(self, user_id: str, server_id: str) -> None: + raise RuntimeError("redis down") + + manager = MCPServerManager(per_user_oauth_token_store=_Store(), per_user_token_cache=_RaisingLegacyCache()) + await manager.invalidate_user_oauth_token_cache("alice", "srv-1") + @pytest.mark.asyncio async def test_resolve_oauth2_headers_no_user_id(self): """Skip lookup entirely when user_api_key_auth has no user_id.""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py index a8a8ee0fbde..a02acc02502 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py @@ -5136,7 +5136,7 @@ def test_stamp_oauth2_flow_ignores_non_oauth2(): assert payload.oauth2_flow is None -async def _run_edit(old_record, updated_record): +async def _run_edit(old_record, updated_record, purge_mock=None): from litellm.proxy.management_endpoints.mcp_management_endpoints import edit_mcp_server server_id = updated_record.server_id @@ -5163,7 +5163,7 @@ async def _run_edit(old_record, updated_record): patch("litellm.proxy.management_endpoints.mcp_management_endpoints.global_mcp_server_manager") as mock_manager, patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.purge_user_oauth_credentials_for_server", - AsyncMock(return_value=1), + purge_mock if purge_mock is not None else AsyncMock(return_value=1), ) as mock_purge, ): mock_manager.update_server = AsyncMock() @@ -5199,6 +5199,20 @@ async def test_edit_mcp_server_skips_purge_when_identity_unchanged(): mock_purge.assert_not_awaited() +@pytest.mark.asyncio +async def test_edit_mcp_server_purge_failure_does_not_fail_the_edit(): + """The purge is best-effort: a purge exception after a successful update must be swallowed and + logged, never turned into an error response for an edit whose primary job already succeeded.""" + server_id = str(uuid.uuid4()) + old = generate_mock_mcp_server_db_record(server_id=server_id, url="https://old.example.com/mcp") + updated = generate_mock_mcp_server_db_record(server_id=server_id, url="https://new.example.com/mcp") + + result, mock_purge = await _run_edit(old, updated, purge_mock=AsyncMock(side_effect=RuntimeError("db down"))) + + assert result.server_id == server_id + mock_purge.assert_awaited_once() + + @pytest.mark.asyncio async def test_edit_mcp_server_snapshot_failure_skips_purge_but_edit_succeeds(): """The pre-update snapshot read is advisory (it only feeds the purge decision); a read failure From b304620311b3f449b84d37d20ebdc84cf8d4cb20 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 14:33:51 -0700 Subject: [PATCH 084/101] fix(ui): compare url and spec_path independently in the OAuth authorization identity The identity used to pick the audience from spec_path only when values.transport was OPENAPI, but the create form keeps transport in component state rather than form values, so spec_path edits on OpenAPI servers never invalidated a held token. Comparing url and spec_path independently mirrors the backend's mcp_oauth_token_identity and fires regardless of whether transport is present. Invalidation now also wipes only credentials; the admin-typed endpoint fields are kept --- .../mcp_tools/create_mcp_server.tsx | 3 +- .../mcp_tools/mcp_server_edit.test.tsx | 27 ++++++++++++++ .../src/components/mcp_tools/types.test.tsx | 27 ++++++++++++++ .../src/components/mcp_tools/types.tsx | 35 +++++++++++-------- 4 files changed, 77 insertions(+), 15 deletions(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx index 24b8e9eaafb..eb48fd02474 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.tsx @@ -239,7 +239,8 @@ const CreateMCPServer: React.FC = ({ // Discard the held browser-authorized token and its tool preview when the authorization identity // changes (or the modal closes). The CLEARED_ON_INVALIDATION form fields (shared with the edit form // via types.tsx) are reset too; whatever the admin just changed (passed via changedValues) is - // re-applied so the invalidation never wipes their in-flight edit. + // re-applied so the invalidation never wipes their in-flight edit. Admin-typed endpoint fields are + // left alone (see CLEARED_ON_INVALIDATION). const clearHeldOAuthToken = (changedValues: Record = {}) => { setOauthAccessToken(null); clearTools(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index 4f3d7b69b01..e5daf90e992 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -511,6 +511,33 @@ describe("MCPServerEdit OAuth token invalidation", () => { expect(mockRemoveToken).toHaveBeenCalledWith("oauth_server_1", undefined); }); + it("keeps the admin's in-flight endpoint edits when the token is invalidated", async () => { + // Regression: invalidation used to form.resetFields the endpoint fields; with the edit Form's + // initialValues that silently reverted an admin-corrected token_url back to the saved (wrong) + // value while still looking plausible. Only credentials (the minted material) may be wiped. + renderOAuthEdit(); + + const tokenUrlInput = screen.getByPlaceholderText("https://example.com/oauth/token"); + await act(async () => { + fireEvent.change(tokenUrlInput, { target: { value: "https://corrected.example.com/token" } }); + }); + + act(() => { + mockOauth.onTokenReceived?.({ access_token: "tok-1" }); + }); + mockOauth.reset.mockClear(); + + const urlInput = screen.getByPlaceholderText("https://your-mcp-server.com"); + await act(async () => { + fireEvent.change(urlInput, { target: { value: "https://moved.example.com/mcp" } }); + }); + + await waitFor(() => expect(mockOauth.reset).toHaveBeenCalled()); + expect((screen.getByPlaceholderText("https://example.com/oauth/token") as HTMLInputElement).value).toBe( + "https://corrected.example.com/token", + ); + }); + it("keeps a session-authorized token on an http to sse switch with the same url", async () => { // Same url means the same resource/audience (RFC 8707): the minted token is still valid, so a // pure transport swap between the two MCP wire protocols must not force a re-authorize. diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx index 846bcfc9e0b..c6faca1fb51 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.test.tsx @@ -7,9 +7,36 @@ import { handleTransport, handleAuth, getMcpOAuthMode, + getOAuthAuthorizationIdentity, + isHeldOAuthTokenStale, oauth2FlowToFormValue, } from "./types"; +describe("getOAuthAuthorizationIdentity", () => { + // Regression: the identity used to pick the audience from spec_path only when values.transport was + // OPENAPI, but the create form keeps transport in component state, so values.transport was absent and + // spec_path edits on OpenAPI servers never invalidated a held token. + it("changes when spec_path changes even when transport is absent from form values", () => { + const authorized = { auth_type: AUTH_TYPE.OAUTH2, spec_path: "https://a.example.com/openapi.json" }; + const edited = { auth_type: AUTH_TYPE.OAUTH2, spec_path: "https://b.example.com/openapi.json" }; + expect(getOAuthAuthorizationIdentity(edited)).not.toBe(getOAuthAuthorizationIdentity(authorized)); + expect(isHeldOAuthTokenStale(edited, getOAuthAuthorizationIdentity(authorized))).toBe(true); + }); + + it("changes when url changes", () => { + const authorized = { auth_type: AUTH_TYPE.OAUTH2, url: "https://a.example.com/mcp" }; + const edited = { auth_type: AUTH_TYPE.OAUTH2, url: "https://b.example.com/mcp" }; + expect(getOAuthAuthorizationIdentity(edited)).not.toBe(getOAuthAuthorizationIdentity(authorized)); + }); + + it("is stable across non-mint fields", () => { + const authorized = { auth_type: AUTH_TYPE.OAUTH2, url: "https://a.example.com/mcp", server_name: "one" }; + const renamed = { auth_type: AUTH_TYPE.OAUTH2, url: "https://a.example.com/mcp", server_name: "two" }; + expect(getOAuthAuthorizationIdentity(renamed)).toBe(getOAuthAuthorizationIdentity(authorized)); + expect(isHeldOAuthTokenStale(renamed, getOAuthAuthorizationIdentity(authorized))).toBe(false); + }); +}); + describe("handleTransport", () => { it("should default to SSE when transport is null", () => { expect(handleTransport(null)).toBe(TRANSPORT.SSE); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx index e894cf4b8dc..3eba8b30968 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/types.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/types.tsx @@ -58,20 +58,24 @@ export const OAUTH_FLOW = { }; // The fields that determine which upstream OAuth token "Authorize & Fetch" mints: the resource/audience -// (url), the OAuth mode/grant (auth_type, oauth_flow_type), the OAuth client and requested scope -// (credentials.client_id / client_secret / scopes), and the authorization-server endpoints -// (authorization_url / token_url / registration_url). Grounded in RFC 8707 / RFC 8693 and the MCP auth -// spec: an access token is bound to exactly this tuple (resource/audience + scope + client + issuer), so +// (url, or spec_path for OpenAPI servers), the OAuth mode/grant (auth_type, oauth_flow_type), the OAuth +// client and requested scope (credentials.client_id / client_secret / scopes), and the authorization-server +// endpoints (authorization_url / token_url / registration_url). Grounded in RFC 8707 / RFC 8693 and the MCP +// auth spec: an access token is bound to exactly this tuple (resource/audience + scope + client + issuer), so // a previously authorized token is stale if and only if this identity changes and must be re-minted. -// Deliberately EXCLUDES: transport (http<->sse on the same url is the same audience; a transport switch -// only matters when it changes the target url, which `target` already captures), delegate_auth_to_upstream -// (a downstream-usage toggle that is never sent to the authorize request), and all metadata/RBAC/routing -// fields. Shared by the create and edit forms so their invalidation logic cannot drift. +// url and spec_path are compared independently rather than selected by transport: the create form keeps +// transport in component state, not in form values, so a transport-conditional target would silently pin the +// audience to a missing url and never fire for spec_path edits on OpenAPI servers. Mirrors the backend's +// mcp_oauth_token_identity. Deliberately EXCLUDES: transport itself (http<->sse on the same url is the same +// audience; a switch to/from OpenAPI shows up as url/spec_path changes because each form clears the field the +// new transport does not use), delegate_auth_to_upstream (a downstream-usage toggle that is never sent to the +// authorize request), and all metadata/RBAC/routing fields. Shared by the create and edit forms so their +// invalidation logic cannot drift. export const getOAuthAuthorizationIdentity = (values: Record): string => { const credentials = (values.credentials ?? {}) as Record; - const target = values.transport === TRANSPORT.OPENAPI ? values.spec_path : values.url; const identity = { - target: typeof target === "string" ? target : null, + url: typeof values.url === "string" ? values.url : null, + spec_path: typeof values.spec_path === "string" ? values.spec_path : null, auth_type: values.auth_type ?? null, oauth_flow_type: values.oauth_flow_type ?? null, client_id: credentials.client_id ?? null, @@ -84,10 +88,13 @@ export const getOAuthAuthorizationIdentity = (values: Record): return JSON.stringify(identity); }; -// The form fields wiped when a held OAuth token is invalidated: the fetched token + DCR client live in -// `credentials`, and the three endpoint fields were discovered by the authorize flow, so all of them are -// stale together with the token. Shared by the create and edit forms so what gets wiped cannot drift. -export const CLEARED_ON_INVALIDATION = ["credentials", "authorization_url", "token_url", "registration_url"] as const; +// The form fields wiped when a held OAuth token is invalidated: only `credentials`, which holds the +// minted material (the fetched token + DCR client). The authorization/token/registration endpoint +// fields are deliberately NOT wiped: nothing programmatic ever writes them (upstream discovery happens +// backend-side), so they only ever hold admin input, and resetting them would wipe it (create) or +// silently revert it to the saved record (edit, whose Form has initialValues). Shared by the create and +// edit forms so what gets wiped cannot drift. +export const CLEARED_ON_INVALIDATION = ["credentials"] as const; // True when a token was authorized in this session (authorizedIdentity recorded at mint time) and the // form's current identity no longer matches it. Every invalidation decision in both forms goes through From 71e0491d37cde6568ca90dca31368d835322bad4 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 15:06:43 -0700 Subject: [PATCH 085/101] fix(ui): preview tools with a staged interactive OAuth token in the edit form For authorization_code the edit preview listed tools by server_id only, relying on the stored per-user DB credential, so a token authorized in the edit session gave an empty preview until the admin saved; the create form previews the identical state through the config-based preview endpoint, which takes the token explicitly. The edit fetch now routes through that same endpoint when a staged interactive token is held, built from the form values with the saved record as fallback, and keeps the by-server_id listing for every other case --- .../mcp_tools/mcp_server_edit.test.tsx | 25 +++++++++ .../components/mcp_tools/mcp_server_edit.tsx | 53 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index e5daf90e992..9a9db2eeb95 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -10,6 +10,7 @@ vi.mock("../networking", () => ({ updateMCPServer: vi.fn(), listMCPTools: vi.fn().mockResolvedValue({ tools: [], error: null }), storeMCPOAuthUserCredential: vi.fn().mockResolvedValue({}), + testMCPToolsListRequest: vi.fn().mockResolvedValue({ tools: [], error: null }), })); vi.mock("../molecules/notifications_manager", () => ({ @@ -511,6 +512,30 @@ describe("MCPServerEdit OAuth token invalidation", () => { expect(mockRemoveToken).toHaveBeenCalledWith("oauth_server_1", undefined); }); + it("previews tools with a staged interactive OAuth token before it is saved", async () => { + // Regression: for authorization_code the fetch went by server_id only, relying on the stored DB + // credential, so a token authorized in this edit session gave an empty preview until the admin + // saved; the create form previews the identical state via the config-based preview endpoint. + mockOauth.tokenResponse = { access_token: "staged-obo-tok" }; + + renderOAuthEdit(); + + await waitFor(() => { + expect(vi.mocked(networking.testMCPToolsListRequest)).toHaveBeenCalledWith( + "access-token", + expect.objectContaining({ server_id: "oauth_server_1", url: "https://example.com/mcp" }), + "staged-obo-tok", + ); + }); + expect(networking.listMCPTools).not.toHaveBeenCalled(); + // Previewing must stay stateless: the staged token is committed only by an explicit Save + // (storeMCPOAuthUserCredential for authorization_code, setToken for the client-forwarded modes). + expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled(); + expect(mockSetToken).not.toHaveBeenCalled(); + expect(networking.updateMCPServer).not.toHaveBeenCalled(); + mockOauth.tokenResponse = null; + }); + it("keeps the admin's in-flight endpoint edits when the token is invalidated", async () => { // Regression: invalidation used to form.resetFields the endpoint fields; with the edit Form's // initialValues that silently reverted an admin-corrected token_url back to the saved (wrong) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index de3528ea6a9..58f63e12019 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -17,7 +17,7 @@ import { getMcpOAuthMode, oauth2FlowToFormValue, } from "./types"; -import { updateMCPServer, listMCPTools, storeMCPOAuthUserCredential } from "../networking"; +import { updateMCPServer, listMCPTools, storeMCPOAuthUserCredential, testMCPToolsListRequest } from "../networking"; import { getToken, isTokenValid, removeToken, setToken } from "@/utils/mcpTokenStore"; import { buildMcpPassthroughAuthHeader } from "@/utils/mcpHeaderUtils"; import MCPServerCostConfig from "./mcp_server_cost_config"; @@ -421,6 +421,53 @@ const MCPServerEdit: React.FC = ({ } }; + // A token authorized in this edit session for interactive OAuth (authorization_code) is only + // committed to the DB on save, so a plain by-server_id listing cannot use it and the preview would + // stay empty until the admin saves; the create form previews the identical state through the + // config-based preview endpoint, which takes the staged token explicitly. Returns false when there + // is no staged interactive token so fetchTools falls through to the by-server_id listing. + const previewWithStagedInteractiveToken = async ( + isPassthrough: boolean, + isBrowserHeldTokenMode: boolean, + ): Promise => { + const stagedToken = + !isPassthrough && !isBrowserHeldTokenMode && getEffectiveAuthType() === AUTH_TYPE.OAUTH2 + ? oauthTokenResponse?.access_token + : undefined; + if (!stagedToken) { + return false; + } + setIsLoadingTools(true); + setToolsError(null); + try { + const values = form.getFieldsValue(true); + const rawTransport = values.transport || mcpServer.transport; + const previewConfig = { + server_id: mcpServer.server_id, + server_name: values.server_name || mcpServer.server_name || mcpServer.alias, + url: values.url || mcpServer.url, + transport: rawTransport === TRANSPORT.OPENAPI ? TRANSPORT.HTTP : rawTransport, + auth_type: AUTH_TYPE.OAUTH2, + authorization_url: values.authorization_url, + token_url: values.token_url, + registration_url: values.registration_url, + }; + const toolsResponse = await testMCPToolsListRequest(accessToken, previewConfig, stagedToken); + if (toolsResponse.tools && !toolsResponse.error) { + setTools(toolsResponse.tools); + } else { + setTools([]); + setToolsError(toolsResponse.message || "Failed to load tools"); + } + } catch (error) { + setTools([]); + setToolsError(error instanceof Error ? error.message : "Failed to load tools"); + } finally { + setIsLoadingTools(false); + } + return true; + }; + const fetchTools = async () => { if (!accessToken || !mcpServer.server_id) return; @@ -436,6 +483,10 @@ const MCPServerEdit: React.FC = ({ delegate_auth_to_upstream: mcpServer.delegate_auth_to_upstream, }) === "passthrough"; const isBrowserHeldTokenMode = isClientForwardedTokenMode(getEffectiveAuthType()); + + if (await previewWithStagedInteractiveToken(isPassthrough, isBrowserHeldTokenMode)) { + return; + } if (isPassthrough || isBrowserHeldTokenMode) { const token = oauthTokenResponse?.access_token ?? From 4786e599b0f78b4d7ad4be96782b24202404c5ed Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 15:28:39 -0700 Subject: [PATCH 086/101] test(ui): pin the client-forwarded token contract on create and edit The create and edit submit paths for true_passthrough and oauth_delegate persist only the tool configuration: the parametrized create test authorizes, disables the allowlist, and asserts nothing is persisted before submit, then that the create payload carries allowed_tools but no credentials and no occurrence of the token anywhere in the serialized payload, no per-user DB credential is written, and the token is committed to sessionStorage only, keyed to the created server. The edit save test gains the same serialized-payload assertion --- .../mcp_tools/create_mcp_server.test.tsx | 62 +++++++++++++++++++ .../mcp_tools/mcp_server_edit.test.tsx | 1 + 2 files changed, 63 insertions(+) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx index 658116ede1f..02374a3ffa4 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/create_mcp_server.test.tsx @@ -374,6 +374,68 @@ describe("CreateMCPServer", () => { expect(credentials.access_token).toBeUndefined(); }); + it.each([ + ["true_passthrough", "True Passthrough (no LiteLLM auth)"], + ["oauth_delegate", "OAuth Delegate (client-supplied upstream token)"], + ])("persists only tool config on create for %s; the token stays browser-held", async (_authType, optionLabel) => { + oauthHook.tokenResponse = { access_token: "upstream-tok", token_type: "Bearer" }; + await selectHttpTransport(); + + const user = userEvent.setup({ delay: null }); + await user.type(getServerNameInput(), "CF_Server"); + await user.type(screen.getByPlaceholderText("https://your-mcp-server.com"), "https://example.com/mcp"); + + await selectAntOption("Authentication", optionLabel); + + await waitFor(() => expect(oauthHook.onTokenReceived).toBeTruthy()); + await act(async () => { + oauthHook.onTokenReceived!({ access_token: "upstream-tok", token_type: "Bearer" }, undefined); + }); + + fireEvent.click(screen.getByRole("button", { name: "Disable all tools" })); + + // Previewing and configuring must stay stateless: nothing is persisted anywhere (server row, + // per-user DB credential, sessionStorage) until the admin submits. + expect(networking.createMCPServer).not.toHaveBeenCalled(); + expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled(); + expect(setToken).not.toHaveBeenCalled(); + + const createdServer = { + server_id: "new-cf-server", + server_name: "CF_Server", + alias: "CF_Server", + url: "https://example.com/mcp", + transport: "http", + auth_type: _authType, + created_at: "2024-01-01T00:00:00Z", + created_by: "user-1", + updated_at: "2024-01-01T00:00:00Z", + updated_by: "user-1", + }; + vi.mocked(networking.createMCPServer).mockResolvedValue(createdServer); + + const submitButton = screen.getByRole("button", { name: "Add MCP Server" }); + await act(async () => { + fireEvent.click(submitButton); + }); + + await waitFor(() => expect(networking.createMCPServer).toHaveBeenCalledTimes(1)); + const [, payload] = vi.mocked(networking.createMCPServer).mock.calls[0]; + + // Only the tool configuration persists on the server row; the upstream token appears nowhere + // in the create payload and no per-user DB credential is written. The token is committed to + // sessionStorage only, keyed to the created server. + expect(payload.allowed_tools).toEqual([]); + expect(payload.credentials).toBeUndefined(); + expect(JSON.stringify(payload)).not.toContain("upstream-tok"); + expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled(); + expect(setToken).toHaveBeenCalledWith( + "new-cf-server", + expect.objectContaining({ access_token: "upstream-tok" }), + undefined, + ); + }); + it("should not show auth value field when None auth type is selected", async () => { await selectHttpTransport(); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index 9a9db2eeb95..93ff1333cd8 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -1339,6 +1339,7 @@ describe("MCPServerEdit (OAuth token persistence on save)", () => { expect(networking.storeMCPOAuthUserCredential).not.toHaveBeenCalled(); const [, payload] = vi.mocked(networking.updateMCPServer).mock.calls[0]; expect(payload.credentials).toBeUndefined(); + expect(JSON.stringify(payload)).not.toContain("cf-tok"); }, ); From c46c9d46526077139c7c864d886950ac40c912e6 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 16:04:19 -0700 Subject: [PATCH 087/101] docs(mcp): mcp_server_resource docstring matches the origin-only redaction The field doc still said scheme + host + path while the redactor now strips the path along with userinfo, query, and fragment, since hosted MCP servers routinely embed the credential in the path --- litellm/types/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6b99cfa3314..c093c213e50 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2533,9 +2533,10 @@ class StandardLoggingMCPToolCall(TypedDict, total=False): mcp_server_resource: Optional[str] """ - The upstream MCP server resource identifier (scheme + host + path) the tool call was - forwarded to. Redacted for logging: userinfo, query string, and fragment are stripped so an - upstream URL carrying an embedded token or secret query parameter never reaches log metadata. + The origin (scheme + host + port) of the upstream MCP server the tool call was forwarded + to. Redacted for logging: userinfo, the path, the query string, and the fragment are all + stripped, because hosted MCP servers routinely embed the credential in the URL path and + this value is readable by callers via request logs. Records which upstream received a relayed request; never a credential. """ From 9dcc21cd48aaee6650e2f8063692d5d1b78a1d41 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 16:18:25 -0700 Subject: [PATCH 088/101] refactor(mcp): batch the purge row deletion into one query The per-row delete_many loop becomes a single delete filtered to the enumerated OAuth users' (user_id IN, server_id) pairs; same rows deleted, same BYOK-sparing precision, same count-mismatch detection, one round-trip instead of N --- litellm/proxy/_experimental/mcp_server/db.py | 4 ++-- .../_experimental/mcp_server/test_db_credentials.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index c6b7620b649..e4f8b0c331d 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1175,8 +1175,8 @@ async def purge_user_oauth_credentials_for_server( oauth_rows = [row for row in rows if _decode_oauth_payload(row.credential_b64) is not None] if not oauth_rows: return 0 - deleted_count = sum( - [await repo.table.delete_many(where={"user_id": row.user_id, "server_id": server_id}) for row in oauth_rows] + deleted_count = await repo.table.delete_many( + where={"server_id": server_id, "user_id": {"in": [row.user_id for row in oauth_rows]}} ) if invalidate_token_cache is None: from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 0bdcffe1530..7269774442b 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -171,7 +171,7 @@ async def test_purge_user_oauth_credentials_for_server_invalidates_each_user(): prisma = MagicMock() prisma.db.litellm_mcpusercredentials.find_many = AsyncMock(return_value=[_oauth_row("alice"), _oauth_row("bob")]) - prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=1) + prisma.db.litellm_mcpusercredentials.delete_many = AsyncMock(return_value=2) invalidations = [] @@ -181,15 +181,17 @@ async def test_purge_user_oauth_credentials_for_server_invalidates_each_user(): purged = await purge_user_oauth_credentials_for_server(prisma, "srv-1", invalidate_token_cache=record_invalidation) assert purged == 2 - assert prisma.db.litellm_mcpusercredentials.delete_many.await_count == 2 + prisma.db.litellm_mcpusercredentials.delete_many.assert_awaited_once_with( + where={"server_id": "srv-1", "user_id": {"in": ["alice", "bob"]}} + ) assert set(invalidations) == {("alice", "srv-1"), ("bob", "srv-1")} @pytest.mark.asyncio async def test_purge_user_oauth_credentials_for_server_spares_byok_rows(): """Regression: the purge used to delete_many on server_id alone, wiping BYOK API keys that share - the LiteLLM_MCPUserCredentials table. Only rows holding an OAuth2 payload may be deleted, each by - its (user_id, server_id) pair, and only their users' token caches invalidated.""" + the LiteLLM_MCPUserCredentials table. Only rows holding an OAuth2 payload may be deleted (one + batched query filtered to their user_ids), and only their users' token caches invalidated.""" from litellm.proxy._experimental.mcp_server.db import purge_user_oauth_credentials_for_server prisma = MagicMock() @@ -205,7 +207,7 @@ async def test_purge_user_oauth_credentials_for_server_spares_byok_rows(): assert purged == 1 prisma.db.litellm_mcpusercredentials.delete_many.assert_awaited_once_with( - where={"user_id": "alice", "server_id": "srv-1"} + where={"server_id": "srv-1", "user_id": {"in": ["alice"]}} ) assert invalidations == [("alice", "srv-1")] From db8c872c7d3cf374b143e1785edc3e8c98194f06 Mon Sep 17 00:00:00 2001 From: Tin Date: Thu, 9 Jul 2026 16:26:30 -0700 Subject: [PATCH 089/101] fix(ui): staged edit preview sends explicit oauth2_flow and spec_path; invalidation clears the tool list The preview endpoint infers client_credentials when the inherited client_id, client_secret, and token_url are all present (common once DCR or discovery filled them) and then strips the forwarded bearer to preview as M2M, so the staged interactive token was silently unused; sending oauth2_flow=authorization_code bypasses the inference. spec_path now rides along so OpenAPI servers take the spec-based preview path the create form gets. clearHeldOAuthToken also empties the tool list, mirroring the create form's clearTools, so a preview fetched with the discarded token never lingers while the refetch is in flight --- .../mcp_tools/mcp_server_edit.test.tsx | 36 ++++++++++++++++++- .../components/mcp_tools/mcp_server_edit.tsx | 7 ++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx index 93ff1333cd8..adb3e161da5 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.test.tsx @@ -523,7 +523,13 @@ describe("MCPServerEdit OAuth token invalidation", () => { await waitFor(() => { expect(vi.mocked(networking.testMCPToolsListRequest)).toHaveBeenCalledWith( "access-token", - expect.objectContaining({ server_id: "oauth_server_1", url: "https://example.com/mcp" }), + // oauth2_flow must be explicit: the preview endpoint infers client_credentials from + // inherited client_id/client_secret/token_url and would strip the staged bearer. + expect.objectContaining({ + server_id: "oauth_server_1", + url: "https://example.com/mcp", + oauth2_flow: "authorization_code", + }), "staged-obo-tok", ); }); @@ -536,6 +542,34 @@ describe("MCPServerEdit OAuth token invalidation", () => { mockOauth.tokenResponse = null; }); + it("previews an OpenAPI server's staged token against its spec_path", async () => { + mockOauth.tokenResponse = { access_token: "staged-obo-tok" }; + + render( + , + ); + + await waitFor(() => { + expect(vi.mocked(networking.testMCPToolsListRequest)).toHaveBeenCalledWith( + "access-token", + expect.objectContaining({ spec_path: "https://example.com/openapi.json" }), + "staged-obo-tok", + ); + }); + mockOauth.tokenResponse = null; + }); + it("keeps the admin's in-flight endpoint edits when the token is invalidated", async () => { // Regression: invalidation used to form.resetFields the endpoint fields; with the edit Form's // initialValues that silently reverted an admin-corrected token_url back to the saved (wrong) diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx index 58f63e12019..7446c96c40e 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_server_edit.tsx @@ -405,6 +405,7 @@ const MCPServerEdit: React.FC = ({ if (mcpServer.server_id) { removeToken(mcpServer.server_id, userID); } + setTools([]); resetOAuthFlow(); form.resetFields([...CLEARED_ON_INVALIDATION]); const preserved = Object.fromEntries( @@ -442,12 +443,18 @@ const MCPServerEdit: React.FC = ({ try { const values = form.getFieldsValue(true); const rawTransport = values.transport || mcpServer.transport; + // oauth2_flow must be explicit: the preview endpoint infers client_credentials from the + // inherited client_id/client_secret/token_url (common once DCR or discovery filled them) and + // would strip the staged bearer to preview as M2M. spec_path keeps OpenAPI servers on the + // spec-based preview path, mirroring the create form's config. const previewConfig = { server_id: mcpServer.server_id, server_name: values.server_name || mcpServer.server_name || mcpServer.alias, url: values.url || mcpServer.url, + spec_path: values.spec_path || mcpServer.spec_path, transport: rawTransport === TRANSPORT.OPENAPI ? TRANSPORT.HTTP : rawTransport, auth_type: AUTH_TYPE.OAUTH2, + oauth2_flow: MCP_OAUTH2_FLOW_INTERACTIVE, authorization_url: values.authorization_url, token_url: values.token_url, registration_url: values.registration_url, From 65d90fd5cfbf1d5690708973948b989d9cbfbb1f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 9 Jul 2026 17:43:33 -0700 Subject: [PATCH 090/101] refactor(ui): colocate 11 route segments' components into _components/ (#32704) Colocation follow-up to the App Router migration: move each page's owned components out of the shared src/components dump and into its route segment's _components/ folder, draining the shared bucket. Convention: a component used by exactly one segment goes in that segment's _components/ (private, matching Next's _ route-exclusion); a component shared by 2+ segments stays in @/components. No new _shared/ folder. Rename-in-place (segment already had a local components/ folder): - api-reference (also relocates the shared CodeBlock, used by playground and cost-tracking, to @/components/CodeBlock) - memory, budgets, access-groups - caching, projects, guardrails-monitor Extract from src/components (page view lived in the shared dump): - AdminPanel -> admin-panel, organizations -> organizations, general_settings -> router-settings, usage -> old-usage Each folder/view was verified to have no importer other than its own page (cross-checked across src, tests, and e2e_tests). Relative imports inside moved single files are rewritten to absolute @/components/*; colocated tests move with their subject and have their vi.mock paths rewritten to match. Grandfathered lint suppressions (tremor, react-hooks, and similar, all pre-existing) are re-keyed to the new paths with counts unchanged. No behavior change. --- ui/litellm-dashboard/eslint-suppressions.json | 48 +++++++++---------- .../AccessGroupsDetailsPage.test.tsx | 0 .../AccessGroupsDetailsPage.tsx | 0 .../AccessGroupsModal/AccessGroupBaseForm.tsx | 0 .../AccessGroupCreateModal.tsx | 0 .../AccessGroupEditModal.tsx | 0 .../AccessGroupsPage.test.tsx | 0 .../AccessGroupsPage.tsx | 0 .../{components => _components}/types.ts | 0 .../app/(dashboard)/access-groups/page.tsx | 2 +- .../_components}/AdminPanel.test.tsx | 14 +++--- .../admin-panel/_components}/AdminPanel.tsx | 24 +++++----- .../src/app/(dashboard)/admin-panel/page.tsx | 2 +- .../APIReferenceView.test.tsx | 2 +- .../{ => _components}/APIReferenceView.tsx | 4 +- .../{components => _components}/DocLink.tsx | 0 .../app/(dashboard)/api-reference/page.tsx | 2 +- .../budget_modal.tsx | 0 .../budget_panel.test.tsx | 0 .../budget_panel.tsx | 0 .../{components => _components}/constants.ts | 0 .../edit_budget_modal.tsx | 0 .../src/app/(dashboard)/budgets/page.tsx | 2 +- .../cache_dashboard.tsx | 0 .../cache_health.tsx | 0 .../cache_settings/CacheFieldSection.tsx | 0 .../cache_settings/CacheFormField.tsx | 0 .../cache_settings/RedisTypeSelector.test.tsx | 0 .../cache_settings/RedisTypeSelector.tsx | 0 .../cache_settings/cacheSettingsFields.ts | 0 .../cache_settings/cacheSettingsUtils.test.ts | 0 .../cache_settings/cacheSettingsUtils.ts | 0 .../cache_settings/index.test.tsx | 0 .../cache_settings/index.tsx | 0 .../response_time_indicator.tsx | 0 .../src/app/(dashboard)/caching/page.tsx | 2 +- .../components/how_it_works.test.tsx | 2 +- .../cost-tracking/components/how_it_works.tsx | 2 +- .../EvaluationSettingsModal.tsx | 0 .../GuardrailConfig.test.tsx | 0 .../GuardrailConfig.tsx | 0 .../GuardrailDetail.tsx | 0 .../GuardrailsMonitorView.test.tsx | 0 .../GuardrailsMonitorView.tsx | 0 .../GuardrailsOverview.tsx | 0 .../ScoreChart.test.tsx | 0 .../ScoreChart.tsx | 0 .../(dashboard)/guardrails-monitor/page.tsx | 2 +- .../MemoryEditModal.tsx | 0 .../MemoryView.tsx | 0 .../src/app/(dashboard)/memory/page.tsx | 2 +- .../old-usage/_components}/usage.tsx | 10 ++-- .../src/app/(dashboard)/old-usage/page.tsx | 2 +- .../_components}/organizations.test.tsx | 4 +- .../_components}/organizations.tsx | 25 ++++++---- .../app/(dashboard)/organizations/page.tsx | 2 +- .../components/chat_ui/AgentBuilderView.tsx | 2 +- .../ProjectDetailsPage.test.tsx | 0 .../ProjectDetailsPage.tsx | 0 .../ProjectKeysSection.test.tsx | 0 .../ProjectKeysSection.tsx | 0 .../ProjectKeysTable.test.tsx | 0 .../ProjectKeysTable.tsx | 0 .../ProjectModals/CreateProjectModal.test.tsx | 0 .../ProjectModals/CreateProjectModal.tsx | 0 .../ProjectModals/EditProjectModal.test.tsx | 0 .../ProjectModals/EditProjectModal.tsx | 0 .../ProjectModals/ProjectBaseForm.test.tsx | 0 .../ProjectModals/ProjectBaseForm.tsx | 0 .../ProjectModals/projectFormUtils.test.ts | 0 .../ProjectModals/projectFormUtils.ts | 0 .../ProjectsPage.test.tsx | 0 .../ProjectsPage.tsx | 0 .../src/app/(dashboard)/projects/page.tsx | 2 +- .../_components}/general_settings.tsx | 8 ++-- .../app/(dashboard)/router-settings/page.tsx | 2 +- .../components/CodeBlock.tsx | 0 .../tests/CreateKeyPage.expiredToken.test.tsx | 8 ++-- 78 files changed, 91 insertions(+), 84 deletions(-) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/AccessGroupsDetailsPage.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/AccessGroupsDetailsPage.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/AccessGroupsModal/AccessGroupBaseForm.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/AccessGroupsModal/AccessGroupCreateModal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/AccessGroupsModal/AccessGroupEditModal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/AccessGroupsPage.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/AccessGroupsPage.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/access-groups/{components => _components}/types.ts (100%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/admin-panel/_components}/AdminPanel.test.tsx (96%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/admin-panel/_components}/AdminPanel.tsx (93%) rename ui/litellm-dashboard/src/app/(dashboard)/api-reference/{ => _components}/APIReferenceView.test.tsx (97%) rename ui/litellm-dashboard/src/app/(dashboard)/api-reference/{ => _components}/APIReferenceView.tsx (97%) rename ui/litellm-dashboard/src/app/(dashboard)/api-reference/{components => _components}/DocLink.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/budgets/{components => _components}/budget_modal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/budgets/{components => _components}/budget_panel.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/budgets/{components => _components}/budget_panel.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/budgets/{components => _components}/constants.ts (100%) rename ui/litellm-dashboard/src/app/(dashboard)/budgets/{components => _components}/edit_budget_modal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_dashboard.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_health.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/CacheFieldSection.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/CacheFormField.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/RedisTypeSelector.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/RedisTypeSelector.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/cacheSettingsFields.ts (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/cacheSettingsUtils.test.ts (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/cacheSettingsUtils.ts (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/index.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/cache_settings/index.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/caching/{components => _components}/response_time_indicator.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/EvaluationSettingsModal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/GuardrailConfig.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/GuardrailConfig.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/GuardrailDetail.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/GuardrailsMonitorView.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/GuardrailsMonitorView.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/GuardrailsOverview.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/ScoreChart.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/{components => _components}/ScoreChart.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/memory/{components => _components}/MemoryEditModal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/memory/{components => _components}/MemoryView.tsx (100%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/old-usage/_components}/usage.tsx (99%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/organizations/_components}/organizations.test.tsx (87%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/organizations/_components}/organizations.tsx (96%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectDetailsPage.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectDetailsPage.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectKeysSection.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectKeysSection.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectKeysTable.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectKeysTable.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/CreateProjectModal.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/CreateProjectModal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/EditProjectModal.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/EditProjectModal.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/ProjectBaseForm.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/ProjectBaseForm.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/projectFormUtils.test.ts (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectModals/projectFormUtils.ts (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectsPage.test.tsx (100%) rename ui/litellm-dashboard/src/app/(dashboard)/projects/{components => _components}/ProjectsPage.tsx (100%) rename ui/litellm-dashboard/src/{components => app/(dashboard)/router-settings/_components}/general_settings.tsx (96%) rename ui/litellm-dashboard/src/{app/(dashboard)/api-reference => }/components/CodeBlock.tsx (100%) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index fe8f182c106..b490cf71768 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -4,32 +4,32 @@ "count": 1 } }, - "src/app/(dashboard)/api-reference/APIReferenceView.tsx": { + "src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/budgets/components/budget_modal.tsx": { + "src/app/(dashboard)/budgets/_components/budget_modal.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/budgets/components/budget_panel.test.tsx": { + "src/app/(dashboard)/budgets/_components/budget_panel.test.tsx": { "unused-imports/no-unused-imports": { "count": 2 } }, - "src/app/(dashboard)/budgets/components/budget_panel.tsx": { + "src/app/(dashboard)/budgets/_components/budget_panel.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/budgets/components/edit_budget_modal.tsx": { + "src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/caching/components/cache_dashboard.tsx": { + "src/app/(dashboard)/caching/_components/cache_dashboard.tsx": { "no-restricted-imports": { "count": 1 }, @@ -40,17 +40,17 @@ "count": 2 } }, - "src/app/(dashboard)/caching/components/cache_health.tsx": { + "src/app/(dashboard)/caching/_components/cache_health.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/caching/components/cache_settings/RedisTypeSelector.tsx": { + "src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/caching/components/cache_settings/index.tsx": { + "src/app/(dashboard)/caching/_components/cache_settings/index.tsx": { "no-restricted-imports": { "count": 1 }, @@ -136,32 +136,32 @@ "count": 2 } }, - "src/app/(dashboard)/guardrails-monitor/components/EvaluationSettingsModal.tsx": { + "src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx": { "react-hooks/set-state-in-effect": { "count": 1 } }, - "src/app/(dashboard)/guardrails-monitor/components/GuardrailDetail.tsx": { + "src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx": { "no-nested-ternary": { "count": 3 } }, - "src/app/(dashboard)/guardrails-monitor/components/GuardrailsMonitorView.tsx": { + "src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx": { "no-restricted-imports": { "count": 1 } }, - "src/app/(dashboard)/guardrails-monitor/components/GuardrailsOverview.tsx": { + "src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx": { "no-nested-ternary": { "count": 8 } }, - "src/app/(dashboard)/guardrails-monitor/components/ScoreChart.test.tsx": { + "src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx": { "react/display-name": { "count": 1 } }, - "src/app/(dashboard)/guardrails-monitor/components/ScoreChart.tsx": { + "src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx": { "no-restricted-imports": { "count": 1 } @@ -326,7 +326,7 @@ "count": 2 } }, - "src/app/(dashboard)/memory/components/MemoryView.tsx": { + "src/app/(dashboard)/memory/_components/MemoryView.tsx": { "react-hooks/set-state-in-effect": { "count": 1 } @@ -522,7 +522,7 @@ "count": 1 } }, - "src/app/(dashboard)/projects/components/ProjectDetailsPage.tsx": { + "src/app/(dashboard)/projects/_components/ProjectDetailsPage.tsx": { "no-nested-ternary": { "count": 3 }, @@ -530,17 +530,17 @@ "count": 1 } }, - "src/app/(dashboard)/projects/components/ProjectKeysSection.tsx": { + "src/app/(dashboard)/projects/_components/ProjectKeysSection.tsx": { "react-hooks/set-state-in-effect": { "count": 1 } }, - "src/app/(dashboard)/projects/components/ProjectModals/ProjectBaseForm.tsx": { + "src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx": { "react-hooks/set-state-in-effect": { "count": 2 } }, - "src/app/(dashboard)/projects/components/ProjectsPage.tsx": { + "src/app/(dashboard)/projects/_components/ProjectsPage.tsx": { "react-hooks/set-state-in-effect": { "count": 1 } @@ -851,7 +851,7 @@ "count": 1 } }, - "src/components/AdminPanel.tsx": { + "src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx": { "no-restricted-imports": { "count": 1 }, @@ -1520,7 +1520,7 @@ "count": 1 } }, - "src/components/general_settings.tsx": { + "src/app/(dashboard)/router-settings/_components/general_settings.tsx": { "no-nested-ternary": { "count": 3 }, @@ -2028,7 +2028,7 @@ "count": 1 } }, - "src/components/organizations.tsx": { + "src/app/(dashboard)/organizations/_components/organizations.tsx": { "no-restricted-imports": { "count": 1 } @@ -2371,7 +2371,7 @@ "count": 1 } }, - "src/components/usage.tsx": { + "src/app/(dashboard)/old-usage/_components/usage.tsx": { "no-restricted-imports": { "count": 2 }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsDetailsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsDetailsPage.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsDetailsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsDetailsPage.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsDetailsPage.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsModal/AccessGroupBaseForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupBaseForm.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsModal/AccessGroupBaseForm.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupBaseForm.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsModal/AccessGroupCreateModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupCreateModal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsModal/AccessGroupCreateModal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupCreateModal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsModal/AccessGroupEditModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsModal/AccessGroupEditModal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsModal/AccessGroupEditModal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/AccessGroupsPage.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/types.ts b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/types.ts similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/access-groups/components/types.ts rename to ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/types.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/page.tsx index ae4712b826e..4e9f7031c6d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { AccessGroupsPage } from "./components/AccessGroupsPage"; +import { AccessGroupsPage } from "./_components/AccessGroupsPage"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function AccessGroups() { diff --git a/ui/litellm-dashboard/src/components/AdminPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.test.tsx similarity index 96% rename from ui/litellm-dashboard/src/components/AdminPanel.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.test.tsx index 7d1d2f46cf1..220db23338e 100644 --- a/ui/litellm-dashboard/src/components/AdminPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.test.tsx @@ -8,34 +8,34 @@ const mockGetAllowedIPs = vi.fn(); const mockAddAllowedIP = vi.fn(); const mockDeleteAllowedIP = vi.fn(); -vi.mock("./networking", () => ({ +vi.mock("@/components/networking", () => ({ getSSOSettings: (...args: unknown[]) => mockGetSSOSettings(...args), getAllowedIPs: (...args: unknown[]) => mockGetAllowedIPs(...args), addAllowedIP: (...args: unknown[]) => mockAddAllowedIP(...args), deleteAllowedIP: (...args: unknown[]) => mockDeleteAllowedIP(...args), })); -vi.mock("./constants", () => ({ +vi.mock("@/components/constants", () => ({ useBaseUrl: () => "http://localhost:4000", })); -vi.mock("./Settings/AdminSettings/SSOSettings/SSOSettings", () => ({ +vi.mock("@/components/Settings/AdminSettings/SSOSettings/SSOSettings", () => ({ default: () =>
SSO Settings
, })); -vi.mock("./Settings/AdminSettings/UISettings/UISettings", () => ({ +vi.mock("@/components/Settings/AdminSettings/UISettings/UISettings", () => ({ default: () =>
UI Settings
, })); -vi.mock("./SCIM", () => ({ +vi.mock("@/components/SCIM", () => ({ default: () =>
SCIM Config
, })); -vi.mock("./SSOModals", () => ({ +vi.mock("@/components/SSOModals", () => ({ default: () =>
SSO Modals
, })); -vi.mock("./UIAccessControlForm", () => ({ +vi.mock("@/components/UIAccessControlForm", () => ({ default: () =>
UI Access Control Form
, })); diff --git a/ui/litellm-dashboard/src/components/AdminPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx similarity index 93% rename from ui/litellm-dashboard/src/components/AdminPanel.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx index 7867c184ed2..611efd6a588 100644 --- a/ui/litellm-dashboard/src/components/AdminPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx @@ -16,18 +16,18 @@ import { } from "@tremor/react"; import { Alert, Button as Button2, Form, Input, Modal, Space, Tabs, Typography } from "antd"; import React, { useEffect, useState } from "react"; -import NewBadge from "./common_components/NewBadge"; -import { useBaseUrl } from "./constants"; -import NotificationsManager from "./molecules/notifications_manager"; -import { addAllowedIP, deleteAllowedIP, getAllowedIPs, getSSOSettings } from "./networking"; -import SCIMConfig from "./SCIM"; -import LoggingSettings from "./Settings/AdminSettings/LoggingSettings/LoggingSettings"; -import SSOSettings from "./Settings/AdminSettings/SSOSettings/SSOSettings"; -import UISettings from "./Settings/AdminSettings/UISettings/UISettings"; -import HashicorpVault from "./Settings/AdminSettings/HashicorpVault/HashicorpVault"; -import PluginSettings from "./Settings/AdminSettings/PluginSettings/PluginSettings"; -import SSOModals from "./SSOModals"; -import UIAccessControlForm from "./UIAccessControlForm"; +import NewBadge from "@/components/common_components/NewBadge"; +import { useBaseUrl } from "@/components/constants"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { addAllowedIP, deleteAllowedIP, getAllowedIPs, getSSOSettings } from "@/components/networking"; +import SCIMConfig from "@/components/SCIM"; +import LoggingSettings from "@/components/Settings/AdminSettings/LoggingSettings/LoggingSettings"; +import SSOSettings from "@/components/Settings/AdminSettings/SSOSettings/SSOSettings"; +import UISettings from "@/components/Settings/AdminSettings/UISettings/UISettings"; +import HashicorpVault from "@/components/Settings/AdminSettings/HashicorpVault/HashicorpVault"; +import PluginSettings from "@/components/Settings/AdminSettings/PluginSettings/PluginSettings"; +import SSOModals from "@/components/SSOModals"; +import UIAccessControlForm from "@/components/UIAccessControlForm"; const { Title, Paragraph, Text } = Typography; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/page.tsx index aac835b02fc..47076acc9f0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/page.tsx @@ -1,6 +1,6 @@ "use client"; -import AdminPanel from "@/components/AdminPanel"; +import AdminPanel from "./_components/AdminPanel"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/APIReferenceView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.test.tsx similarity index 97% rename from ui/litellm-dashboard/src/app/(dashboard)/api-reference/APIReferenceView.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.test.tsx index a73973bd742..66fa0dfa63f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/APIReferenceView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.test.tsx @@ -2,7 +2,7 @@ import { render } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; import APIReferenceView from "./APIReferenceView"; -vi.mock("./components/CodeBlock", () => ({ +vi.mock("@/components/CodeBlock", () => ({ __esModule: true, default: ({ code }: { code: string }) => 
{code}
, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/APIReferenceView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx similarity index 97% rename from ui/litellm-dashboard/src/app/(dashboard)/api-reference/APIReferenceView.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx index 5861cc87e5b..333bd1cad13 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/APIReferenceView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx @@ -1,8 +1,8 @@ "use client"; import React from "react"; import { Text, Tab, TabGroup, TabList, TabPanel, TabPanels, Grid } from "@tremor/react"; -import CodeBlock from "./components/CodeBlock"; -import DocLink from "@/app/(dashboard)/api-reference/components/DocLink"; +import CodeBlock from "@/components/CodeBlock"; +import DocLink from "./DocLink"; interface ApiRefProps { proxySettings: { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/components/DocLink.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/api-reference/components/DocLink.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/DocLink.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx index 42cf094f0bb..d7c977b0870 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/page.tsx @@ -1,6 +1,6 @@ "use client"; -import APIReferenceView from "@/app/(dashboard)/api-reference/APIReferenceView"; +import APIReferenceView from "./_components/APIReferenceView"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import useProxySettings from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_modal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/budgets/components/budget_panel.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/constants.ts b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/constants.ts similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/budgets/components/constants.ts rename to ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/constants.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/components/edit_budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/budgets/components/edit_budget_modal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx index 547699411e7..ca34589a679 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx @@ -1,6 +1,6 @@ "use client"; -import BudgetPanel from "./components/budget_panel"; +import BudgetPanel from "./_components/budget_panel"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function Budgets() { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_dashboard.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_health.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_health.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/CacheFieldSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/CacheFieldSection.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFieldSection.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/CacheFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/CacheFormField.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/CacheFormField.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/RedisTypeSelector.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/RedisTypeSelector.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/RedisTypeSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/RedisTypeSelector.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/RedisTypeSelector.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/cacheSettingsFields.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/cacheSettingsFields.ts rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsFields.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/cacheSettingsUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/cacheSettingsUtils.test.ts rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/cacheSettingsUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/cacheSettingsUtils.ts rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/cacheSettingsUtils.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/index.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/cache_settings/index.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_settings/index.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/components/response_time_indicator.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/response_time_indicator.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/caching/components/response_time_indicator.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/caching/_components/response_time_indicator.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/page.tsx index 0ef88ec9eb5..33f3e81c689 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/page.tsx @@ -1,6 +1,6 @@ "use client"; -import CacheDashboard from "./components/cache_dashboard"; +import CacheDashboard from "./_components/cache_dashboard"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function Caching() { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.test.tsx index 711a8795f15..a574f4b628e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.test.tsx @@ -5,7 +5,7 @@ import userEvent from "@testing-library/user-event"; import { renderWithProviders } from "../../../../../tests/test-utils"; import HowItWorks from "./how_it_works"; -vi.mock("@/app/(dashboard)/api-reference/components/CodeBlock", () => ({ +vi.mock("@/components/CodeBlock", () => ({ default: ({ code }: { code: string }) => 
{code}
, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.tsx index 79abf6baa31..5fa27551d16 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/components/how_it_works.tsx @@ -1,6 +1,6 @@ import React, { useState, useMemo } from "react"; import { Text, TextInput } from "@tremor/react"; -import CodeBlock from "@/app/(dashboard)/api-reference/components/CodeBlock"; +import CodeBlock from "@/components/CodeBlock"; const HowItWorks: React.FC = () => { const [responseCost, setResponseCost] = useState(""); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/EvaluationSettingsModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/EvaluationSettingsModal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailConfig.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailConfig.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailConfig.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailDetail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailDetail.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailsMonitorView.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailsMonitorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailsMonitorView.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/GuardrailsOverview.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/ScoreChart.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/ScoreChart.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/ScoreChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/components/ScoreChart.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.tsx index 388ed168f17..0c4e69c2d80 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/page.tsx @@ -1,6 +1,6 @@ "use client"; -import GuardrailsMonitorView from "./components/GuardrailsMonitorView"; +import GuardrailsMonitorView from "./_components/GuardrailsMonitorView"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function GuardrailsMonitor() { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryEditModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryEditModal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryEditModal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryEditModal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/memory/components/MemoryView.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/memory/_components/MemoryView.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/memory/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/memory/page.tsx index 031a027d518..b88996c5396 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/memory/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/memory/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { MemoryView } from "./components/MemoryView"; +import { MemoryView } from "./_components/MemoryView"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx similarity index 99% rename from ui/litellm-dashboard/src/components/usage.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx index 91c12fd1fa2..01f8cb1cd45 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx @@ -14,9 +14,9 @@ import { import React, { useState, useEffect } from "react"; -import ViewUserSpend from "./view_user_spend"; -import { ProxySettings } from "./user_dashboard"; -import UsageDatePicker from "./shared/usage_date_picker"; +import ViewUserSpend from "@/components/view_user_spend"; +import { ProxySettings } from "@/components/user_dashboard"; +import UsageDatePicker from "@/components/shared/usage_date_picker"; import { Grid, Col, @@ -48,8 +48,8 @@ import { adminGlobalActivity, adminGlobalActivityPerModel, getProxyUISettings, -} from "./networking"; -import TopKeyView from "./UsagePage/components/EntityUsage/TopKeyView"; +} from "@/components/networking"; +import TopKeyView from "@/components/UsagePage/components/EntityUsage/TopKeyView"; import { MoneyCell } from "@/components/shared/table_cells"; import { formatNumberWithCommas } from "@/utils/dataUtils"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx index cc1f2c35e44..138dd97e5e8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx @@ -1,6 +1,6 @@ "use client"; -import Usage from "@/components/usage"; +import Usage from "./_components/usage"; import { DeprecationBanner } from "@/components/DeprecationBanner"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; diff --git a/ui/litellm-dashboard/src/components/organizations.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.test.tsx similarity index 87% rename from ui/litellm-dashboard/src/components/organizations.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.test.tsx index 9be31be6170..75a6d30ac2e 100644 --- a/ui/litellm-dashboard/src/components/organizations.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.test.tsx @@ -3,11 +3,11 @@ import { render } from "@testing-library/react"; import React from "react"; import { describe, expect, it, vi } from "vitest"; -vi.mock("./vector_store_management/VectorStoreSelector", () => ({ +vi.mock("@/components/vector_store_management/VectorStoreSelector", () => ({ __esModule: true, default: () => null, })); -vi.mock("./mcp_server_management/MCPServerSelector", () => ({ +vi.mock("@/components/mcp_server_management/MCPServerSelector", () => ({ __esModule: true, default: () => null, })); diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.tsx similarity index 96% rename from ui/litellm-dashboard/src/components/organizations.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.tsx index edebc17087a..d3af5b62668 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/organizations.tsx @@ -28,16 +28,21 @@ import { Form, Input, Modal, Select as Select2, Tooltip } from "antd"; import { useQueryClient } from "@tanstack/react-query"; import React, { useState } from "react"; import { DateCell, IdCell, MoneyCell } from "@/components/shared/table_cells"; -import DeleteResourceModal from "./common_components/DeleteResourceModal"; -import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; -import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; -import MCPServerSelector from "./mcp_server_management/MCPServerSelector"; -import { ModelSelect } from "./ModelSelect/ModelSelect"; -import NotificationsManager from "./molecules/notifications_manager"; -import { Organization, organizationCreateCall, organizationDeleteCall, organizationListCall } from "./networking"; -import OrganizationInfoView from "./organization/organization_view"; -import NumericalInput from "./shared/numerical_input"; -import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; +import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; +import TableIconActionButton from "@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { getModelDisplayName } from "@/components/key_team_helpers/fetch_available_models_team_key"; +import MCPServerSelector from "@/components/mcp_server_management/MCPServerSelector"; +import { ModelSelect } from "@/components/ModelSelect/ModelSelect"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { + Organization, + organizationCreateCall, + organizationDeleteCall, + organizationListCall, +} from "@/components/networking"; +import OrganizationInfoView from "@/components/organization/organization_view"; +import NumericalInput from "@/components/shared/numerical_input"; +import VectorStoreSelector from "@/components/vector_store_management/VectorStoreSelector"; interface OrganizationsTableProps { userRole: string; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx index 87e0faf9cce..649e54f63eb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/page.tsx @@ -1,6 +1,6 @@ "use client"; -import OrganizationsTable from "@/components/organizations"; +import OrganizationsTable from "./_components/organizations"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function OrganizationsPage() { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx index 35f5dcf06c0..d4333b95c62 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx @@ -11,7 +11,7 @@ import { } from "@ant-design/icons"; import { Button, Input, Modal, Select, Spin, Tabs } from "antd"; import React, { useCallback, useEffect, useState } from "react"; -import CodeBlock from "@/app/(dashboard)/api-reference/components/CodeBlock"; +import CodeBlock from "@/components/CodeBlock"; import NotificationsManager from "@/components/molecules/notifications_manager"; import { keyCreateCall, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectDetailsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectDetailsPage.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectDetailsPage.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectDetailsPage.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectDetailsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectDetailsPage.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectDetailsPage.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectDetailsPage.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysSection.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysSection.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysSection.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysSection.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysSection.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysSection.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysTable.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysTable.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysTable.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectKeysTable.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectKeysTable.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/CreateProjectModal.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/CreateProjectModal.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/CreateProjectModal.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/CreateProjectModal.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/CreateProjectModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/CreateProjectModal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/CreateProjectModal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/CreateProjectModal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/EditProjectModal.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/EditProjectModal.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/EditProjectModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/EditProjectModal.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/EditProjectModal.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/ProjectBaseForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/ProjectBaseForm.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/ProjectBaseForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/ProjectBaseForm.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/ProjectBaseForm.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/projectFormUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormUtils.test.ts similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/projectFormUtils.test.ts rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormUtils.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/projectFormUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormUtils.ts similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectModals/projectFormUtils.ts rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectModals/projectFormUtils.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.test.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/projects/components/ProjectsPage.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/page.tsx index 62b67118109..2ba014592c9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { ProjectsPage } from "./components/ProjectsPage"; +import { ProjectsPage } from "./_components/ProjectsPage"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function Projects() { diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx similarity index 96% rename from ui/litellm-dashboard/src/components/general_settings.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 038547c6e0e..3955e80f5e9 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -13,14 +13,14 @@ import { Switch, } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; -import { getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting } from "./networking"; +import { getGeneralSettingsCall, updateConfigFieldSetting, deleteConfigFieldSetting } from "@/components/networking"; import { InputNumber } from "antd"; import { TrashIcon } from "@heroicons/react/outline"; import { StatusBadge } from "@/components/shared/table_cells"; -import RouterSettings from "./router_settings"; -import Fallbacks from "./Settings/RouterSettings/Fallbacks/Fallbacks"; -import RoutingGroups from "./routing_groups"; +import RouterSettings from "@/components/router_settings"; +import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"; +import RoutingGroups from "@/components/routing_groups"; interface GeneralSettingsPageProps { accessToken: string | null; userRole: string | null; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/page.tsx index 46029b529ec..90f41ac58a4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/page.tsx @@ -1,6 +1,6 @@ "use client"; -import GeneralSettings from "@/components/general_settings"; +import GeneralSettings from "./_components/general_settings"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function RouterSettingsPage() { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/components/CodeBlock.tsx b/ui/litellm-dashboard/src/components/CodeBlock.tsx similarity index 100% rename from ui/litellm-dashboard/src/app/(dashboard)/api-reference/components/CodeBlock.tsx rename to ui/litellm-dashboard/src/components/CodeBlock.tsx diff --git a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx index 660c49fff77..07ed1cb5c2e 100644 --- a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx +++ b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx @@ -108,13 +108,15 @@ vi.mock("@/components/navbar", () => ({ default: stub("navbar") })); vi.mock("@/components/user_dashboard", () => ({ default: stub("user-dashboard") })); vi.mock("@/components/templates/model_dashboard", () => ({ default: stub("model-dashboard") })); vi.mock("@/components/teams", () => ({ default: stub("teams") })); -vi.mock("@/components/organizations", () => ({ +vi.mock("@/app/(dashboard)/organizations/_components/organizations", () => ({ default: stub("organizations"), fetchOrganizations: vi.fn(), // consumed in effects })); vi.mock("@/components/admins", () => ({ default: stub("admin-panel") })); vi.mock("@/components/settings", () => ({ default: stub("settings") })); -vi.mock("@/components/general_settings", () => ({ default: stub("general-settings") })); +vi.mock("@/app/(dashboard)/router-settings/_components/general_settings", () => ({ + default: stub("general-settings"), +})); vi.mock("@/components/pass_through_settings", () => ({ default: stub("pass-through-settings") })); vi.mock("@/components/budgets/budget_panel", () => ({ default: stub("budget-panel") })); vi.mock("@/components/view_logs", () => ({ default: stub("spend-logs") })); @@ -123,7 +125,7 @@ vi.mock("@/components/new_usage", () => ({ default: stub("new-usage") })); vi.mock("@/components/api_ref", () => ({ default: stub("api-ref") })); vi.mock("@/components/chat_ui/ChatUI", () => ({ default: stub("chat-ui") })); vi.mock("@/components/leftnav", () => ({ default: stub("sidebar") })); -vi.mock("@/components/usage", () => ({ default: stub("usage") })); +vi.mock("@/app/(dashboard)/old-usage/_components/usage", () => ({ default: stub("usage") })); vi.mock("@/components/cache_dashboard", () => ({ default: stub("cache-dashboard") })); vi.mock("@/components/guardrails", () => ({ default: stub("guardrails") })); vi.mock("@/components/prompts", () => ({ default: stub("prompts") })); From 592510ec18b880fc5bea533af18afa317dd1e67d Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 9 Jul 2026 18:18:52 -0700 Subject: [PATCH 091/101] feat(ui): shadcn charts foundation with tremor-compatible wrappers (#32668) --- ui/litellm-dashboard/eslint-metrics.json | 4 +- ui/litellm-dashboard/eslint-suppressions.json | 74 ++-- ui/litellm-dashboard/package-lock.json | 221 ++++++++++-- ui/litellm-dashboard/package.json | 1 + .../_components/ScoreChart.test.tsx | 38 +- .../_components/ScoreChart.tsx | 50 +-- .../shared/charts/area_chart.test.tsx | 36 ++ .../components/shared/charts/area_chart.tsx | 92 +++++ .../shared/charts/bar_chart.test.tsx | 119 +++++++ .../components/shared/charts/bar_chart.tsx | 119 +++++++ .../shared/charts/chart_legend.test.tsx | 32 ++ .../components/shared/charts/chart_legend.tsx | 25 ++ .../shared/charts/chart_tooltip.test.tsx | 101 ++++++ .../shared/charts/chart_tooltip.tsx | 97 ++++++ .../src/components/shared/charts/colors.ts | 58 ++++ .../shared/charts/donut_chart.test.tsx | 38 ++ .../components/shared/charts/donut_chart.tsx | 67 ++++ .../src/components/shared/charts/index.ts | 12 + .../src/components/ui/card.tsx | 86 +++++ .../src/components/ui/chart.test.tsx | 38 ++ .../src/components/ui/chart.tsx | 324 ++++++++++++++++++ .../src/components/ui/ref-forwarding.test.tsx | 42 +++ ui/litellm-dashboard/tests/setupTests.ts | 37 +- 23 files changed, 1582 insertions(+), 129 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/charts/area_chart.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/bar_chart.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/chart_legend.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/chart_legend.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/colors.ts create mode 100644 ui/litellm-dashboard/src/components/shared/charts/donut_chart.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/donut_chart.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/charts/index.ts create mode 100644 ui/litellm-dashboard/src/components/ui/card.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/chart.test.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/chart.tsx diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index d69b3e1f729..2e204c63a48 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,6 +1,6 @@ { - "@typescript-eslint/no-explicit-any": 1980, - "complexity": 128, + "@typescript-eslint/no-explicit-any": 1978, + "complexity": 129, "local/no-large-inline-object-arg": 512, "local/no-long-condition-chain": 233, "max-depth": 59, diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index b490cf71768..32ab92cbcc9 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -4,6 +4,14 @@ "count": 1 } }, + "src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx": { + "no-restricted-imports": { + "count": 1 + }, + "react-hooks/set-state-in-effect": { + "count": 1 + } + }, "src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx": { "no-restricted-imports": { "count": 1 @@ -156,16 +164,6 @@ "count": 8 } }, - "src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx": { - "react/display-name": { - "count": 1 - } - }, - "src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/hooks/accessGroups/useAccessGroupDetails.ts": { "no-restricted-syntax": { "count": 1 @@ -373,6 +371,22 @@ "count": 1 } }, + "src/app/(dashboard)/old-usage/_components/usage.tsx": { + "no-restricted-imports": { + "count": 2 + }, + "react-hooks/immutability": { + "count": 1 + }, + "react-hooks/purity": { + "count": 1 + } + }, + "src/app/(dashboard)/organizations/_components/organizations.tsx": { + "no-restricted-imports": { + "count": 1 + } + }, "src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx": { "no-restricted-imports": { "count": 1 @@ -649,6 +663,14 @@ "count": 1 } }, + "src/app/(dashboard)/router-settings/_components/general_settings.tsx": { + "no-nested-ternary": { + "count": 3 + }, + "no-restricted-imports": { + "count": 2 + } + }, "src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx": { "no-restricted-imports": { "count": 1 @@ -851,14 +873,6 @@ "count": 1 } }, - "src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx": { - "no-restricted-imports": { - "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 1 - } - }, "src/components/CreateUserButton.tsx": { "no-restricted-imports": { "count": 1 @@ -1520,14 +1534,6 @@ "count": 1 } }, - "src/app/(dashboard)/router-settings/_components/general_settings.tsx": { - "no-nested-ternary": { - "count": 3 - }, - "no-restricted-imports": { - "count": 2 - } - }, "src/components/guardrails.tsx": { "react-hooks/set-state-in-effect": { "count": 1 @@ -2028,11 +2034,6 @@ "count": 1 } }, - "src/app/(dashboard)/organizations/_components/organizations.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/page_utils.test.ts": { "max-nested-callbacks": { "count": 3 @@ -2371,17 +2372,6 @@ "count": 1 } }, - "src/app/(dashboard)/old-usage/_components/usage.tsx": { - "no-restricted-imports": { - "count": 2 - }, - "react-hooks/immutability": { - "count": 1 - }, - "react-hooks/purity": { - "count": 1 - } - }, "src/components/user_agent_activity.tsx": { "no-restricted-imports": { "count": 2 diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index ae3660f59e9..56c0a4f9500 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -34,6 +34,7 @@ "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-syntax-highlighter": "15.6.6", + "recharts": "3.9.2", "remark-gfm": "4.0.1", "tailwind-merge": "3.4.0", "uuid": "14.0.0" @@ -2927,6 +2928,32 @@ "npm": ">=9.5.0" } }, + "node_modules/@reduxjs/toolkit": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz", + "integrity": "sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==", + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@standard-schema/utils": "^0.3.0", + "immer": "^11.0.0", + "redux": "^5.0.1", + "redux-thunk": "^3.1.0", + "reselect": "^5.1.0" + }, + "peerDependencies": { + "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", + "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-redux": { + "optional": true + } + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.61.1", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.61.1.tgz", @@ -3284,6 +3311,18 @@ "dev": true, "license": "MIT" }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@standard-schema/utils": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz", + "integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==", + "license": "MIT" + }, "node_modules/@swc/helpers": { "version": "0.5.15", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", @@ -3804,6 +3843,42 @@ "react-dom": ">=16.6.0" } }, + "node_modules/@tremor/react/node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/@tremor/react/node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, + "node_modules/@tremor/react/node_modules/recharts": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.4.tgz", + "integrity": "sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==", + "deprecated": "1.x and 2.x branches are no longer active. Bump to Recharts v3 to receive latest features and bugfixes. See https://github.com/recharts/recharts/wiki/3.0-migration-guide", + "license": "MIT", + "dependencies": { + "clsx": "^2.0.0", + "eventemitter3": "^4.0.1", + "lodash": "^4.17.21", + "react-is": "^18.3.1", + "react-smooth": "^4.0.4", + "recharts-scale": "^0.4.4", + "tiny-invariant": "^1.3.1", + "victory-vendor": "^36.6.8" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@tremor/react/node_modules/tailwind-merge": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.6.1.tgz", @@ -3814,6 +3889,28 @@ "url": "https://github.com/sponsors/dcastil" } }, + "node_modules/@tremor/react/node_modules/victory-vendor": { + "version": "36.9.2", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", + "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", + "license": "MIT AND ISC", + "dependencies": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, "node_modules/@tybys/wasm-util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", @@ -4069,6 +4166,12 @@ "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", "license": "MIT" }, + "node_modules/@types/use-sync-external-store": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz", + "integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==", + "license": "MIT" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.60.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.60.1.tgz", @@ -6309,6 +6412,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es-toolkit": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz", + "integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", @@ -6872,9 +6985,9 @@ } }, "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", "license": "MIT" }, "node_modules/expect-type": { @@ -6901,9 +7014,9 @@ "license": "MIT" }, "node_modules/fast-equals": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.4.0.tgz", - "integrity": "sha512-jt2DW/aNFNwke7AUd+Z+e6pz39KO5rzdbbFCg2sGafS4mk13MI7Z8O5z9cADNn5lhGODIgLwug6TZO2ctf7kcw==", + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.4.1.tgz", + "integrity": "sha512-DjlFSM5Pk9cGcL0q5QXl66eGzx0N6szNgaswwc5ZphlBohjTVJSnGgI+rJVOgOi65qUoQnDZN4nDqi33udtydQ==", "license": "MIT", "engines": { "node": ">=6.0.0" @@ -7688,6 +7801,16 @@ "node": ">= 4" } }, + "node_modules/immer": { + "version": "11.1.11", + "resolved": "https://registry.npmjs.org/immer/-/immer-11.1.11.tgz", + "integrity": "sha512-qzXuyXAkPySAGYkfsAwodDPWT8Zm7/Uo5BNt4BjhMhG5WlWyZZ4wQqnWwdS8kjlQ1Cwu6gjw3A6+0gTQwlyYtw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/immer" + } + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -11589,7 +11712,6 @@ "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", - "dev": true, "license": "MIT" }, "node_modules/react-json-view-lite": { @@ -11631,6 +11753,29 @@ "react": ">=18" } }, + "node_modules/react-redux": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.3.0.tgz", + "integrity": "sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==", + "license": "MIT", + "dependencies": { + "@types/use-sync-external-store": "^0.0.6", + "use-sync-external-store": "^1.4.0" + }, + "peerDependencies": { + "@types/react": "^18.2.25 || ^19", + "react": "^18.0 || ^19", + "redux": "^5.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "redux": { + "optional": true + } + } + }, "node_modules/react-smooth": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.4.tgz", @@ -11707,26 +11852,33 @@ } }, "node_modules/recharts": { - "version": "2.15.4", - "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.15.4.tgz", - "integrity": "sha512-UT/q6fwS3c1dHbXv2uFgYJ9BMFHu3fwnd7AYZaEQhXuYQ4hgsxLvsUXzGdKeZrW5xopzDCvuA2N41WJ88I7zIw==", + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.9.2.tgz", + "integrity": "sha512-G4fy+Pk46RaXgwWMh+Nzhyo/lbFAVqXo9gtetlyehe6Ehge9CsgDuOTwQDD+i1+llaLktNBiNq4bhnGlDRXFtw==", "license": "MIT", + "workspaces": [ + "www" + ], "dependencies": { - "clsx": "^2.0.0", - "eventemitter3": "^4.0.1", - "lodash": "^4.17.21", - "react-is": "^18.3.1", - "react-smooth": "^4.0.4", - "recharts-scale": "^0.4.4", - "tiny-invariant": "^1.3.1", - "victory-vendor": "^36.6.8" + "@reduxjs/toolkit": "^1.9.0 || 2.x.x", + "clsx": "^2.1.1", + "decimal.js-light": "^2.5.1", + "es-toolkit": "^1.39.3", + "eventemitter3": "^5.0.1", + "immer": "^11.1.8", + "react-redux": "8.x.x || 9.x.x", + "reselect": "5.2.0", + "tiny-invariant": "^1.3.3", + "use-sync-external-store": "^1.2.2", + "victory-vendor": "^37.0.2" }, "engines": { - "node": ">=14" + "node": ">=18" }, "peerDependencies": { - "react": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "node_modules/recharts-scale": { @@ -11738,12 +11890,6 @@ "decimal.js-light": "^2.4.1" } }, - "node_modules/recharts/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "license": "MIT" - }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -11758,6 +11904,21 @@ "node": ">=8" } }, + "node_modules/redux": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", + "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", + "license": "MIT" + }, + "node_modules/redux-thunk": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz", + "integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==", + "license": "MIT", + "peerDependencies": { + "redux": "^5.0.0" + } + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -13429,9 +13590,9 @@ } }, "node_modules/victory-vendor": { - "version": "36.9.2", - "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", - "integrity": "sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ==", + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", + "integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==", "license": "MIT AND ISC", "dependencies": { "@types/d3-array": "^3.0.3", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 1b0ce315e4d..1747a40da56 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -50,6 +50,7 @@ "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-syntax-highlighter": "15.6.6", + "recharts": "3.9.2", "remark-gfm": "4.0.1", "tailwind-merge": "3.4.0", "uuid": "14.0.0" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx index 3a36eb9621e..dba34ea9a86 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.test.tsx @@ -1,35 +1,9 @@ import React from "react"; -import { describe, it, expect, vi } from "vitest"; +import { describe, it, expect } from "vitest"; import { screen } from "@testing-library/react"; import { renderWithProviders } from "../../../../../tests/test-utils"; import { ScoreChart } from "./ScoreChart"; -vi.mock("@tremor/react", async (importOriginal) => { - const actual = await importOriginal(); - // Re-apply the global Button/Tooltip overrides from tests/setupTests.ts. A file-level - // vi.mock fully replaces the setup-level mock, so without this the real Tremor Button - // leaks through and its useTooltip(300) schedules a native setTimeout that can fire - // post-teardown -> "window is not defined". - return { - ...actual, - BarChart: ({ data, categories }: { data: any[]; categories: string[] }) => ( -
- {data.map((d, i) => ( - - {d.date}: {categories.map((c) => `${c}=${d[c]}`).join(", ")} - - ))} -
- ), - Button: React.forwardRef(({ children, ...props }, ref) => ( - - )), - Tooltip: ({ children }: { children?: React.ReactNode }) => <>{children}, - }; -}); - describe("ScoreChart", () => { it("should render the title", () => { renderWithProviders(); @@ -55,10 +29,14 @@ describe("ScoreChart", () => { { date: "2026-03-02", passed: 15, blocked: 1 }, ]; - renderWithProviders(); + const { container } = renderWithProviders(); expect(screen.queryByText("No chart data for this period")).not.toBeInTheDocument(); - expect(screen.getByText(/2026-03-01/)).toBeInTheDocument(); - expect(screen.getByText(/2026-03-02/)).toBeInTheDocument(); + expect(screen.getByText("passed")).toBeInTheDocument(); + expect(screen.getByText("blocked")).toBeInTheDocument(); + expect(screen.getAllByText(/2026-03-01/).length).toBeGreaterThan(0); + expect(screen.getAllByText(/2026-03-02/).length).toBeGreaterThan(0); + const bars = container.querySelectorAll(".recharts-bar"); + expect(bars).toHaveLength(2); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx index daa6054a552..bc11a6fd3e0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx @@ -1,9 +1,10 @@ -import { BarChart, Card, Title } from "@tremor/react"; import React from "react"; +import { BarChart } from "@/components/shared/charts"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; /** * Overview chart: Request Outcomes Over Time (passed vs blocked). - * Uses Tremor BarChart with stacked data. Data from usage/overview API (chart array). + * Stacked bar chart. Data from usage/overview API (chart array). */ interface ScoreChartProps { data?: Array<{ date: string; passed: number; blocked: number }>; @@ -13,26 +14,31 @@ export function ScoreChart({ data }: ScoreChartProps) { const chartData = data && data.length > 0 ? data : []; return ( - - Request Outcomes Over Time -
- {chartData.length > 0 ? ( - v.toLocaleString()} - yAxisWidth={48} - showLegend={true} - stack={true} - /> - ) : ( -
- No chart data for this period -
- )} -
+ + + Request Outcomes Over Time + + +
+ {chartData.length > 0 ? ( + v.toLocaleString()} + yAxisWidth={48} + showLegend={true} + stack={true} + className="h-full" + /> + ) : ( +
+ No chart data for this period +
+ )} +
+ ); } diff --git a/ui/litellm-dashboard/src/components/shared/charts/area_chart.test.tsx b/ui/litellm-dashboard/src/components/shared/charts/area_chart.test.tsx new file mode 100644 index 00000000000..cd033c5ce27 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/area_chart.test.tsx @@ -0,0 +1,36 @@ +import { render } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it } from "vitest"; +import { AreaChart } from "./area_chart"; + +const data = [ + { date: "2026-03-01", tokens: 100, requests: 10 }, + { date: "2026-03-02", tokens: 150, requests: 12 }, +]; + +describe("AreaChart", () => { + it("renders one area per category with the mapped stroke colors", () => { + const { container } = render( + , + ); + + const curves = Array.from(container.querySelectorAll("path.recharts-area-curve")); + expect(curves).toHaveLength(2); + const strokes = new Set(curves.map((curve) => curve.getAttribute("stroke"))); + expect(strokes).toEqual(new Set(["var(--color-blue-500, #3b82f6)", "var(--color-cyan-500, #06b6d4)"])); + }); + + it("renders a fade-out gradient fill per category", () => { + const { container } = render( + , + ); + + const gradients = container.querySelectorAll("defs linearGradient"); + expect(gradients).toHaveLength(2); + const areas = Array.from(container.querySelectorAll("path.recharts-area-area")); + expect(areas).toHaveLength(2); + for (const area of areas) { + expect(area.getAttribute("fill")).toMatch(/^url\(#fill-/); + } + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx b/ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx new file mode 100644 index 00000000000..794baa13cf7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/area_chart.tsx @@ -0,0 +1,92 @@ +"use client"; + +import * as React from "react"; +import { Area, AreaChart as RechartsAreaChart, CartesianGrid, XAxis, YAxis } from "recharts"; +import { ChartContainer, ChartLegend, ChartLegendContent, ChartTooltip, type ChartConfig } from "@/components/ui/chart"; +import { cn } from "@/lib/cva.config"; +import { ValueTooltip, type ChartTooltipComponent } from "./chart_tooltip"; +import { categoryFills, type ChartColor } from "./colors"; + +export type AreaChartProps> = { + data: readonly TDatum[]; + index: string; + categories: readonly string[]; + colors?: readonly ChartColor[]; + valueFormatter?: (value: number) => string; + yAxisWidth?: number; + showLegend?: boolean; + showGridLines?: boolean; + showTooltip?: boolean; + customTooltip?: ChartTooltipComponent; + className?: string; + style?: React.CSSProperties; +}; + +export function AreaChart>({ + data, + index, + categories, + colors, + valueFormatter, + yAxisWidth = 56, + showLegend = true, + showGridLines = true, + showTooltip = true, + customTooltip, + className, + style, +}: AreaChartProps) { + const gradientId = React.useId().replace(/:/g, ""); + const fills = categoryFills(categories.length, colors); + const config: ChartConfig = Object.fromEntries(categories.map((category) => [category, { label: category }])); + const TooltipContent = customTooltip ?? ValueTooltip; + + return ( + + + + {categories.map((category, i) => ( + + + + + ))} + + {showGridLines && } + + + {showTooltip && ( + ( + + )} + /> + )} + {showLegend && ( + } + /> + )} + {categories.map((category, i) => ( + + ))} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/charts/bar_chart.test.tsx b/ui/litellm-dashboard/src/components/shared/charts/bar_chart.test.tsx new file mode 100644 index 00000000000..d5253c86c6f --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/bar_chart.test.tsx @@ -0,0 +1,119 @@ +import { fireEvent, render, screen } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it, vi } from "vitest"; +import { BarChart } from "./bar_chart"; + +const data = [ + { date: "2026-03-01", passed: 10, blocked: 2 }, + { date: "2026-03-02", passed: 15, blocked: 1 }, +]; + +describe("BarChart", () => { + it("renders one bar series per category with the mapped tremor colors", () => { + const { container } = render( + , + ); + + const rectangles = Array.from(container.querySelectorAll("path.recharts-rectangle")); + expect(rectangles).toHaveLength(4); + const fills = new Set(rectangles.map((rect) => rect.getAttribute("fill"))); + expect(fills).toEqual(new Set(["var(--color-green-500, #22c55e)", "var(--color-red-500, #ef4444)"])); + }); + + it("falls back to the tremor default color cycle when no colors are passed", () => { + const { container } = render(); + + const fills = new Set( + Array.from(container.querySelectorAll("path.recharts-rectangle")).map((rect) => rect.getAttribute("fill")), + ); + expect(fills).toEqual(new Set(["var(--color-blue-500, #3b82f6)", "var(--color-cyan-500, #06b6d4)"])); + }); + + it("fires onValueChange with the datum and clicked category", () => { + const onValueChange = vi.fn(); + const { container } = render( + , + ); + + const firstRect = container.querySelector("path.recharts-rectangle"); + expect(firstRect).not.toBeNull(); + fireEvent.click(firstRect!); + + expect(onValueChange).toHaveBeenCalledTimes(1); + const expectedClickItem = { + date: "2026-03-01", + passed: 10, + blocked: 2, + categoryClicked: "passed", + }; + expect(onValueChange).toHaveBeenCalledWith(expectedClickItem); + }); + + it("renders category labels on the y axis in vertical layout", () => { + render( + , + ); + + expect(screen.getAllByText("alpha").length).toBeGreaterThan(0); + expect(screen.getAllByText("beta").length).toBeGreaterThan(0); + }); + + it("applies valueFormatter to the value axis ticks", () => { + render( + `${v} req`} + />, + ); + + expect(screen.getAllByText(/ req$/).length).toBeGreaterThan(0); + }); + + it("renders a legend by default, matching tremor, and hides it when showLegend is false", () => { + const { container, rerender } = render( + , + ); + expect(screen.getByText("passed")).toBeInTheDocument(); + expect(container.querySelector(".recharts-legend-wrapper")).not.toBeNull(); + + rerender(); + expect(screen.queryByText("passed")).not.toBeInTheDocument(); + }); + + it("emits no per-chart style tag; colors flow through fills, not CSS vars", () => { + const { container } = render( + , + ); + expect(container.querySelector("style")).toBeNull(); + }); + + it("stacks bars into a single column per index when stack is set", () => { + const { container } = render( + , + ); + + const xPositions = Array.from(container.querySelectorAll("path.recharts-rectangle")).map( + (rect) => rect.getAttribute("d")?.split(",")[0], + ); + expect(new Set(xPositions).size).toBe(2); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx b/ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx new file mode 100644 index 00000000000..6ee3319dc10 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/bar_chart.tsx @@ -0,0 +1,119 @@ +"use client"; + +import * as React from "react"; +import { Bar, BarChart as RechartsBarChart, CartesianGrid, XAxis, YAxis } from "recharts"; +import { ChartContainer, ChartLegend, ChartLegendContent, ChartTooltip, type ChartConfig } from "@/components/ui/chart"; +import { cn } from "@/lib/cva.config"; +import { ValueTooltip, type ChartTooltipComponent } from "./chart_tooltip"; +import { categoryFills, type ChartColor } from "./colors"; + +export type BarChartProps> = { + data: readonly TDatum[]; + index: string; + categories: readonly string[]; + colors?: readonly ChartColor[]; + valueFormatter?: (value: number) => string; + stack?: boolean; + layout?: "horizontal" | "vertical"; + yAxisWidth?: number; + tickGap?: number; + showLegend?: boolean; + showXAxis?: boolean; + showGridLines?: boolean; + showTooltip?: boolean; + customTooltip?: ChartTooltipComponent; + onValueChange?: (item: TDatum & { categoryClicked: string }) => void; + className?: string; + style?: React.CSSProperties; +}; + +export function BarChart>({ + data, + index, + categories, + colors, + valueFormatter, + stack = false, + layout = "horizontal", + yAxisWidth = 56, + tickGap = 5, + showLegend = true, + showXAxis = true, + showGridLines = true, + showTooltip = true, + customTooltip, + onValueChange, + className, + style, +}: BarChartProps) { + const fills = categoryFills(categories.length, colors); + const config: ChartConfig = Object.fromEntries(categories.map((category) => [category, { label: category }])); + const vertical = layout === "vertical"; + const TooltipContent = customTooltip ?? ValueTooltip; + + return ( + + + {showGridLines && } + {vertical ? ( + + ) : ( + + )} + {vertical ? ( + + ) : ( + + )} + {showTooltip && ( + ( + + )} + /> + )} + {showLegend && ( + } + /> + )} + {categories.map((category, i) => ( + { + if (item.payload) onValueChange({ ...item.payload, categoryClicked: category }); + } + : undefined + } + /> + ))} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/charts/chart_legend.test.tsx b/ui/litellm-dashboard/src/components/shared/charts/chart_legend.test.tsx new file mode 100644 index 00000000000..889927aca43 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/chart_legend.test.tsx @@ -0,0 +1,32 @@ +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it } from "vitest"; +import { CustomLegend } from "./chart_legend"; + +describe("CustomLegend", () => { + it("renders title-cased category names without the metrics prefix", () => { + render(); + + expect(screen.getByText("Total Tokens")).toBeInTheDocument(); + expect(screen.getByText("Spend")).toBeInTheDocument(); + }); + + it("matches colors to categories by index with theme-var values", () => { + const { container } = render( + , + ); + + const dots = Array.from(container.querySelectorAll('span[style*="background-color"]')); + expect(dots[0]?.getAttribute("style")).toContain("--color-blue-500"); + expect(dots[1]?.getAttribute("style")).toContain("--color-green-500"); + }); + + it("cycles colors when there are more categories than colors", () => { + const { container } = render( + , + ); + + const dots = Array.from(container.querySelectorAll('span[style*="background-color"]')); + expect(dots[2]?.getAttribute("style")).toContain("--color-blue-500"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/charts/chart_legend.tsx b/ui/litellm-dashboard/src/components/shared/charts/chart_legend.tsx new file mode 100644 index 00000000000..da252d8bf63 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/chart_legend.tsx @@ -0,0 +1,25 @@ +"use client"; + +import * as React from "react"; +import { formatCategoryName } from "./chart_tooltip"; +import { chartColorValue, type ChartColor } from "./colors"; + +export const CustomLegend = ({ + categories, + colors, +}: { + categories: readonly string[]; + colors: readonly ChartColor[]; +}) => ( +
+ {categories.map((category, idx) => ( +
+ +

{formatCategoryName(category)}

+
+ ))} +
+); diff --git a/ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.test.tsx b/ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.test.tsx new file mode 100644 index 00000000000..7afc7532760 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.test.tsx @@ -0,0 +1,101 @@ +import { render, screen } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it } from "vitest"; +import { CustomTooltip, ValueTooltip, type ChartTooltipProps } from "./chart_tooltip"; + +const metricsPayload = ( + dataKey: string, + value: number, + color = "#3b82f6", +): NonNullable[number] => + ({ + dataKey, + value, + color, + payload: { + date: "2026-01-15", + metrics: { + total_tokens: 1000, + prompt_tokens: 600, + completion_tokens: 400, + spend: 1234.567, + api_requests: 10, + }, + }, + }) as NonNullable[number]; + +describe("CustomTooltip", () => { + it("returns null when not active or payload is empty", () => { + const inactive = render( + , + ); + expect(inactive.container.firstChild).toBeNull(); + + const empty = render(); + expect(empty.container.firstChild).toBeNull(); + }); + + it("renders the label and title-cased category names without the metrics prefix", () => { + render(); + + expect(screen.getByText("2026-01-15")).toBeInTheDocument(); + expect(screen.getByText("Total Tokens")).toBeInTheDocument(); + expect(screen.getByText("1,000")).toBeInTheDocument(); + }); + + it("formats spend values as dollars with two decimals", () => { + render(); + + expect(screen.getByText("$1,234.57")).toBeInTheDocument(); + }); + + it("shows N/A for metrics missing from the row payload", () => { + render(); + + expect(screen.getByText("N/A")).toBeInTheDocument(); + }); + + it("uses the series color for the indicator dot", () => { + const { container } = render( + , + ); + + const dot = container.querySelector('span[style*="background-color"]'); + expect(dot?.getAttribute("style")).toContain("--color-blue-500"); + }); +}); + +describe("ValueTooltip", () => { + const payload = [ + { + dataKey: "passed", + name: "passed", + value: 1000, + color: "#22c55e", + payload: { date: "2026-01-15", passed: 1000 }, + } as NonNullable[number], + ]; + + it("returns null when not active", () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it("renders label, series name, and locale-formatted value by default", () => { + render(); + + expect(screen.getByText("2026-01-15")).toBeInTheDocument(); + expect(screen.getByText("passed")).toBeInTheDocument(); + expect(screen.getByText("1,000")).toBeInTheDocument(); + }); + + it("applies the valueFormatter to values", () => { + render( `$${v}`} />); + + expect(screen.getByText("$1000")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.tsx b/ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.tsx new file mode 100644 index 00000000000..2644b8f720c --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/chart_tooltip.tsx @@ -0,0 +1,97 @@ +"use client"; + +import * as React from "react"; +import type { TooltipContentProps, TooltipValueType } from "recharts"; + +export type ChartTooltipProps = Pick< + TooltipContentProps, + "active" | "payload" | "label" +>; + +export type ChartTooltipComponent = React.ComponentType; + +export const formatCategoryName = (name: string): string => + name + .replace("metrics.", "") + .replace(/_/g, " ") + .split(" ") + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" "); + +export const ValueTooltip = ({ + active, + payload, + label, + valueFormatter, +}: ChartTooltipProps & { valueFormatter?: (value: number) => string }) => { + if (!active || !payload || payload.length === 0) return null; + + const formatValue = (value: unknown): string => { + if (typeof value === "number") return valueFormatter ? valueFormatter(value) : value.toLocaleString(); + return value == null ? "" : String(value); + }; + + return ( +
+ {label != null &&

{String(label)}

} +
+ {payload.map((item, idx) => ( +
+
+ + {String(item.name ?? item.dataKey ?? "")} +
+ {formatValue(item.value)} +
+ ))} +
+
+ ); +}; + +const rawMetricValue = (row: unknown, dataKey: string): number | undefined => { + if (typeof row !== "object" || row === null || !("metrics" in row)) return undefined; + const metrics = (row as { metrics: unknown }).metrics; + if (typeof metrics !== "object" || metrics === null) return undefined; + const metricKey = dataKey.substring(dataKey.indexOf(".") + 1); + const value = (metrics as Record)[metricKey]; + return typeof value === "number" ? value : undefined; +}; + +const formatMetricValue = (rawValue: number | undefined, isSpend: boolean): string => { + if (rawValue === undefined) return "N/A"; + if (isSpend) return `$${rawValue.toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; + return rawValue.toLocaleString(); +}; + +export const CustomTooltip = ({ active, payload, label }: ChartTooltipProps) => { + if (!active || !payload || payload.length === 0) return null; + + return ( +
+

{label == null ? "" : String(label)}

+ {payload.map((item) => { + const dataKey = item.dataKey?.toString(); + if (!dataKey || !item.payload) return null; + + const formattedValue = formatMetricValue(rawMetricValue(item.payload, dataKey), dataKey.includes("spend")); + + return ( +
+
+ +

{formatCategoryName(dataKey)}

+
+

{formattedValue}

+
+ ); + })} +
+ ); +}; diff --git a/ui/litellm-dashboard/src/components/shared/charts/colors.ts b/ui/litellm-dashboard/src/components/shared/charts/colors.ts new file mode 100644 index 00000000000..c30f58e9e4d --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/colors.ts @@ -0,0 +1,58 @@ +export const CHART_COLOR_HEX = { + slate: "#64748b", + gray: "#6b7280", + zinc: "#71717a", + neutral: "#737373", + stone: "#78716c", + red: "#ef4444", + orange: "#f97316", + amber: "#f59e0b", + yellow: "#eab308", + lime: "#84cc16", + green: "#22c55e", + emerald: "#10b981", + teal: "#14b8a6", + cyan: "#06b6d4", + sky: "#0ea5e9", + blue: "#3b82f6", + indigo: "#6366f1", + violet: "#8b5cf6", + purple: "#a855f7", + fuchsia: "#d946ef", + pink: "#ec4899", + rose: "#f43f5e", +} as const; + +export type ChartColor = keyof typeof CHART_COLOR_HEX; + +export const DEFAULT_COLOR_CYCLE: readonly ChartColor[] = [ + "blue", + "cyan", + "sky", + "indigo", + "violet", + "purple", + "fuchsia", + "slate", + "gray", + "zinc", + "neutral", + "stone", + "red", + "orange", + "amber", + "yellow", + "lime", + "green", + "emerald", + "teal", + "pink", + "rose", +]; + +export const chartColorValue = (color: ChartColor): string => `var(--color-${color}-500, ${CHART_COLOR_HEX[color]})`; + +export const categoryFills = (count: number, colors?: readonly ChartColor[]): readonly string[] => { + const cycle = colors && colors.length > 0 ? colors : DEFAULT_COLOR_CYCLE; + return Array.from({ length: count }, (_, i) => chartColorValue(cycle[i % cycle.length])); +}; diff --git a/ui/litellm-dashboard/src/components/shared/charts/donut_chart.test.tsx b/ui/litellm-dashboard/src/components/shared/charts/donut_chart.test.tsx new file mode 100644 index 00000000000..123c6cad0ec --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/donut_chart.test.tsx @@ -0,0 +1,38 @@ +import { render } from "@testing-library/react"; +import React from "react"; +import { describe, expect, it } from "vitest"; +import { DonutChart } from "./donut_chart"; + +const data = [ + { provider: "openai", spend: 40 }, + { provider: "anthropic", spend: 30 }, + { provider: "bedrock", spend: 20 }, +]; + +describe("DonutChart", () => { + it("renders one sector per datum, cycling the given colors", () => { + const { container } = render( + , + ); + + const sectors = Array.from(container.querySelectorAll(".recharts-pie-sector path")); + expect(sectors).toHaveLength(3); + expect(sectors.map((sector) => sector.getAttribute("fill"))).toEqual([ + "var(--color-cyan-500, #06b6d4)", + "var(--color-blue-500, #3b82f6)", + "var(--color-cyan-500, #06b6d4)", + ]); + }); + + it("renders a full pie when variant is pie and a hollow donut otherwise", () => { + const { container: donut } = render(); + const { container: pie } = render( + , + ); + + const donutPath = donut.querySelector(".recharts-pie-sector path")?.getAttribute("d") ?? ""; + const piePath = pie.querySelector(".recharts-pie-sector path")?.getAttribute("d") ?? ""; + expect(donutPath).not.toEqual(piePath); + expect((donutPath.match(/A/g) ?? []).length).toBeGreaterThan((piePath.match(/A/g) ?? []).length); + }); +}); diff --git a/ui/litellm-dashboard/src/components/shared/charts/donut_chart.tsx b/ui/litellm-dashboard/src/components/shared/charts/donut_chart.tsx new file mode 100644 index 00000000000..c2ce8c02e35 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/donut_chart.tsx @@ -0,0 +1,67 @@ +"use client"; + +import * as React from "react"; +import { Cell, Pie, PieChart } from "recharts"; +import { ChartContainer, ChartTooltip, type ChartConfig } from "@/components/ui/chart"; +import { cn } from "@/lib/cva.config"; +import { ValueTooltip } from "./chart_tooltip"; +import { categoryFills, type ChartColor } from "./colors"; + +export type DonutChartProps> = { + data: readonly TDatum[]; + index: string; + category: string; + colors?: readonly ChartColor[]; + variant?: "donut" | "pie"; + valueFormatter?: (value: number) => string; + showTooltip?: boolean; + className?: string; + style?: React.CSSProperties; +}; + +export function DonutChart>({ + data, + index, + category, + colors, + variant = "donut", + valueFormatter, + showTooltip = true, + className, + style, +}: DonutChartProps) { + const fills = categoryFills(data.length, colors); + const config: ChartConfig = Object.fromEntries( + data.map((datum, i) => { + const name = String(datum[index] ?? i); + return [name, { label: name }]; + }), + ); + + return ( + + + {showTooltip && ( + ( + + )} + /> + )} + + {data.map((datum, i) => ( + + ))} + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/charts/index.ts b/ui/litellm-dashboard/src/components/shared/charts/index.ts new file mode 100644 index 00000000000..ba0a7544ddb --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/charts/index.ts @@ -0,0 +1,12 @@ +export { AreaChart, type AreaChartProps } from "./area_chart"; +export { BarChart, type BarChartProps } from "./bar_chart"; +export { CustomLegend } from "./chart_legend"; +export { + CustomTooltip, + ValueTooltip, + formatCategoryName, + type ChartTooltipComponent, + type ChartTooltipProps, +} from "./chart_tooltip"; +export { CHART_COLOR_HEX, DEFAULT_COLOR_CYCLE, categoryFills, chartColorValue, type ChartColor } from "./colors"; +export { DonutChart, type DonutChartProps } from "./donut_chart"; diff --git a/ui/litellm-dashboard/src/components/ui/card.tsx b/ui/litellm-dashboard/src/components/ui/card.tsx new file mode 100644 index 00000000000..3fc0aa65264 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/card.tsx @@ -0,0 +1,86 @@ +import * as React from "react"; + +import { cn } from "@/lib/cva.config"; + +const Card = React.forwardRef & { size?: "default" | "sm" }>( + ({ className, size = "default", ...props }, ref) => ( +
img:first-child]:pt-0 data-[size=sm]:[--card-spacing:--spacing(4)] *:[img:first-child]:rounded-t-xl *:[img:last-child]:rounded-b-xl", + className, + )} + {...props} + /> + ), +); +Card.displayName = "Card"; + +const CardHeader = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +CardHeader.displayName = "CardHeader"; + +const CardTitle = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +CardTitle.displayName = "CardTitle"; + +const CardDescription = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +CardDescription.displayName = "CardDescription"; + +const CardAction = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +CardAction.displayName = "CardAction"; + +const CardContent = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +CardContent.displayName = "CardContent"; + +const CardFooter = React.forwardRef>( + ({ className, ...props }, ref) => ( +
+ ), +); +CardFooter.displayName = "CardFooter"; + +export { Card, CardHeader, CardFooter, CardTitle, CardAction, CardDescription, CardContent }; diff --git a/ui/litellm-dashboard/src/components/ui/chart.test.tsx b/ui/litellm-dashboard/src/components/ui/chart.test.tsx new file mode 100644 index 00000000000..8b70a6e3246 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/chart.test.tsx @@ -0,0 +1,38 @@ +import { render } from "@testing-library/react"; +import * as React from "react"; +import { describe, expect, it } from "vitest"; +import { ChartContainer } from "./chart"; + +describe("ChartStyle hardening", () => { + it("sanitizes config keys and strips structural characters from color values", () => { + const { container } = render( + " }, + }} + > + + , + ); + + const style = container.querySelector("style"); + expect(style).not.toBeNull(); + const css = style!.innerHTML; + + expect(css).toContain("--color-metrics_total_tokens: var(--color-blue-500, #3b82f6);"); + expect(css).not.toContain("metrics.total_tokens"); + expect(css).toContain("--color-evil_key:"); + expect(css).not.toContain("<"); + expect((css.match(/{/g) ?? []).length).toBe((css.match(/}/g) ?? []).length); + }); + + it("emits no style tag when no config entry has a color", () => { + const { container } = render( + + + , + ); + expect(container.querySelector("style")).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ui/chart.tsx b/ui/litellm-dashboard/src/components/ui/chart.tsx new file mode 100644 index 00000000000..14e10b9f06f --- /dev/null +++ b/ui/litellm-dashboard/src/components/ui/chart.tsx @@ -0,0 +1,324 @@ +"use client"; + +import * as React from "react"; +import * as RechartsPrimitive from "recharts"; +import type { TooltipValueType } from "recharts"; + +import { cn } from "@/lib/cva.config"; + +// Format: { THEME_NAME: CSS_SELECTOR } +const THEMES = { light: "", dark: ".dark" } as const; + +const INITIAL_DIMENSION = { width: 320, height: 200 } as const; +type TooltipNameType = number | string; + +export type ChartConfig = Record< + string, + { + label?: React.ReactNode; + icon?: React.ComponentType; + } & ({ color?: string; theme?: never } | { color?: never; theme: Record }) +>; + +type ChartContextProps = { + config: ChartConfig; +}; + +const ChartContext = React.createContext(null); + +function useChart() { + const context = React.useContext(ChartContext); + + if (!context) { + throw new Error("useChart must be used within a "); + } + + return context; +} + +const ChartContainer = React.forwardRef< + HTMLDivElement, + React.ComponentPropsWithoutRef<"div"> & { + config: ChartConfig; + children: React.ComponentProps["children"]; + initialDimension?: { + width: number; + height: number; + }; + } +>(({ id, className, children, config, initialDimension = INITIAL_DIMENSION, ...props }, ref) => { + const uniqueId = React.useId(); + const chartId = `chart-${id ?? uniqueId.replace(/:/g, "")}`; + + return ( + +
+ + + {children} + +
+ + ); +}); +ChartContainer.displayName = "ChartContainer"; + +const cssVarName = (key: string) => key.replace(/[^a-zA-Z0-9_-]/g, "_"); +const cssColorValue = (color: string) => color.replace(/[;{}<>]/g, ""); + +const ChartStyle = ({ id, config }: { id: string; config: ChartConfig }) => { + const colorConfig = Object.entries(config).filter(([, config]) => config.theme ?? config.color); + + if (!colorConfig.length) { + return null; + } + + return ( +