From 31683e0e529cb2cce5584157caa5cd9d3c54c4f5 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Thu, 17 Sep 2026 17:28:44 -0700 Subject: [PATCH] fix(mcp): recover keyed authorization and isolate concurrent code claims --- .../mcp_server/auth/user_api_key_auth_mcp.py | 28 +++++- litellm/proxy/_experimental/mcp_server/db.py | 7 +- .../mcp_server/keyed_oauth_flow.py | 2 +- .../proxy/_experimental/mcp_server/server.py | 9 +- .../auth/test_user_api_key_auth_mcp.py | 80 +++++++++++++++++ .../mcp_server/test_db_credentials.py | 11 +++ .../mcp_server/test_keyed_oauth_flow.py | 86 ++++++++++++++++++- 7 files changed, 215 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 754befd1170..494bc46334e 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -543,7 +543,33 @@ class MCPRequestHandler: from litellm.proxy._experimental.mcp_server.keyed_oauth_flow import validate_keyed_bearer - validated_user_api_key_auth = await validate_keyed_bearer(request, validated_user_api_key_auth) + try: + validated_user_api_key_auth = await validate_keyed_bearer(request, validated_user_api_key_auth) + except HTTPException as exc: + if ( + exc.status_code == 401 + and validated_user_api_key_auth.via_virtual_key + and request_route.rstrip("/") == "/mcp" + ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.server import ( + _raise_preemptive_401_for_unauthenticated_servers, # pyright: ignore[reportPrivateUsage] # reuse the authorized MCP discovery challenge + ) + + allowed_for_recovery: Final = await global_mcp_server_manager.get_allowed_mcp_servers( + validated_user_api_key_auth + ) + await _raise_preemptive_401_for_unauthenticated_servers( + scope=scope, + mcp_servers=mcp_servers, + oauth2_headers=None, + mcp_server_auth_headers=None, + user_api_key_auth=validated_user_api_key_auth, + client_ip=IPAddressUtils.get_mcp_client_ip(request), + allowed_server_ids=frozenset(allowed_for_recovery), + force_keyed_authorization=True, + ) + raise # Leak-defense (single chokepoint): a gateway admission credential (session bearer or bridge # envelope) is NEVER a valid upstream token. Scrub it from EVERY egress context so no diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 074e45ad101..bcfae49909c 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -2163,8 +2163,11 @@ class KeyedOAuthGrantStore: } return await self._table.update_many(where=where, data=data) == 1 - async def revoke(self, grant_id: str) -> None: - where: Final[prisma_db_types.LiteLLM_MCPKeyedOAuthGrantWhereInput] = {"id": grant_id} + async def revoke(self, grant_id: str, *, expected_status: KeyedGrantStatus | None = None) -> None: + where: Final[prisma_db_types.LiteLLM_MCPKeyedOAuthGrantWhereInput] = { + "id": grant_id, + **({"status": expected_status} if expected_status else {}), + } data: Final[prisma_db_types.LiteLLM_MCPKeyedOAuthGrantUpdateManyMutationInput] = { "status": "revoked", "refresh_hash": None, diff --git a/litellm/proxy/_experimental/mcp_server/keyed_oauth_flow.py b/litellm/proxy/_experimental/mcp_server/keyed_oauth_flow.py index fda26cd2e54..258592cb22e 100644 --- a/litellm/proxy/_experimental/mcp_server/keyed_oauth_flow.py +++ b/litellm/proxy/_experimental/mcp_server/keyed_oauth_flow.py @@ -433,7 +433,7 @@ async def exchange_keyed_token( if not await store.transition( grant_id, "code", "exchanging", _expires(120), expected_hash=hash_token(code or "") ): - await store.revoke(grant_id) + await store.revoke(grant_id, expected_status="active") return _oauth_error(400, "invalid_grant", "Authorization code already used or expired") upstream: Final = await exchange_token_with_server( request=request, diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 43a1eac7b61..04ade5a02e4 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -13,7 +13,7 @@ import time import traceback import types import uuid -from collections.abc import AsyncIterator, Callable, Mapping, Sequence +from collections.abc import AsyncIterator, Callable, Collection, Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol from urllib.parse import urlencode @@ -4021,8 +4021,9 @@ if MCP_AVAILABLE: mcp_server_auth_headers: dict[str, dict[str, str]] | None, user_api_key_auth: UserAPIKeyAuth | None, client_ip: str | None, - allowed_server_ids: set[str] | None = None, + allowed_server_ids: Collection[str] | None = None, raw_headers: Mapping[str, str] | None = None, + force_keyed_authorization: bool = False, ) -> None: """Fail fast with HTTP 401 for MCP servers that need user auth but didn't receive it on this request. Covers both gateway-managed OAuth2 @@ -4077,7 +4078,9 @@ if MCP_AVAILABLE: # authorization server is the gateway itself, vaulting via the # authorize interlude); the per-server relay advertised below # cannot vault without a litellm key on its token request. - if await global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth): + if not force_keyed_authorization and await global_mcp_server_manager.has_user_oauth_token( + server, user_api_key_auth + ): continue if _is_mcp_admitted_user_subject(user_api_key_auth): diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py index 90ce821d62e..5aae5b2e1cc 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/auth/test_user_api_key_auth_mcp.py @@ -9431,3 +9431,83 @@ class TestScopedSessionAdmission: def test_scope_field_cannot_be_forged_through_construction(self): forged = UserAPIKeyAuth(user_id="u1", mcp_session_resource_server_id="any-server") assert forged.mcp_session_resource_server_id is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fault", ("revoked", "expired", "malformed", "database", "denied")) +async def test_rejected_keyed_bearer_restarts_original_key_flow_without_sso(monkeypatch, fault): + import time + from types import SimpleNamespace + from urllib.parse import parse_qs, urlsplit + from starlette.requests import Request + from litellm.proxy._experimental.mcp_server import keyed_oauth_flow, mcp_server_manager + from litellm.proxy._experimental.mcp_server.auth import user_api_key_auth_mcp + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + monkeypatch.setenv("LITELLM_SALT_KEY", "keyed-recovery-test-salt") + monkeypatch.setenv("PROXY_BASE_URL", "http://localhost:4000") + server = MCPServer( + server_id="recovery-server", + alias="interactive", + name="interactive", + transport="http", + auth_type="oauth2", + oauth2_flow="authorization_code", + client_id="configured-app", + authorization_url="https://upstream.example/authorize", + token_url="https://upstream.example/token", + ) + manager = mcp_server_manager.global_mcp_server_manager + monkeypatch.setitem(manager.registry, server.server_id, server) + allowed = AsyncMock(return_value=[] if fault == "denied" else [server.server_id]) + monkeypatch.setattr(manager, "get_allowed_mcp_servers", allowed) + monkeypatch.setattr(manager, "has_user_oauth_token", AsyncMock(return_value=True)) + original = UserAPIKeyAuth(user_id="owner", models=["restricted-model"]) + original.via_virtual_key = True + monkeypatch.setattr(user_api_key_auth_mcp, "user_api_key_auth", AsyncMock(return_value=original)) + store = SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace(status="revoked"), + side_effect=RuntimeError("private-db-detail") if fault == "database" else None, + ) + ) + monkeypatch.setattr(keyed_oauth_flow, "_store", lambda: store) + token = keyed_oauth_flow._seal( + keyed_oauth_flow.KEYED_ACCESS_PREFIX, + keyed_oauth_flow.KeyedToken( + kind="access", grant_id="grant", jti="access", exp=0 if fault == "expired" else int(time.time()) + 600 + ), + ) + if fault == "malformed": + token = "llm_kaccess_invalid" + scope = { + "type": "http", + "method": "POST", + "scheme": "http", + "path": "/mcp", + "headers": [ + (b"host", b"localhost:4000"), + (b"x-litellm-api-key", b"Bearer sk-owned"), + (b"x-mcp-servers", b"interactive"), + (b"authorization", f"Bearer {token}".encode()), + ], + } + with pytest.raises(HTTPException) as rejected: + await MCPRequestHandler.process_mcp_request(scope) + if fault == "database": + assert rejected.value.status_code == 503 + assert "private-db-detail" not in str(rejected.value.detail) + allowed.assert_not_awaited() + elif fault == "denied": + assert rejected.value.status_code == 401 + assert not rejected.value.headers + else: + assert rejected.value.status_code == 401 + assert rejected.value.headers and "www-authenticate" in rejected.value.headers + metadata = rejected.value.headers["www-authenticate"].split('resource_metadata="')[1].rstrip('"') + query = parse_qs(urlsplit(metadata).query) + assert query["mcp_server_name"] == ["interactive"] + opened = keyed_oauth_flow.open_keyed_oauth_flow(Request(scope), query["flow"][0]) + assert opened.user_id == "owner" and opened.server_id == server.server_id + assert allowed.call_args.args[0] is original + assert original.models == ["restricted-model"] diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index f52b6e60684..d89d37f20c0 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -1802,3 +1802,14 @@ async def test_keyed_grant_never_trusts_stale_replica_after_revocation(writer_do else: assert await store.get("revoked-grant") is authoritative reader_table.find_unique.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_code_replay_revokes_only_previously_issued_tokens() -> None: + from litellm.proxy._experimental.mcp_server.db import KeyedOAuthGrantStore + + table: Final = SimpleNamespace(update_many=AsyncMock(return_value=0)) + store: Final = KeyedOAuthGrantStore(SimpleNamespace(db=SimpleNamespace(litellm_mcpkeyedoauthgrant=table))) + await store.revoke("grant", expected_status="active") + assert table.update_many.call_args.kwargs["where"] == {"id": "grant", "status": "active"} + assert table.update_many.call_args.kwargs["data"] == {"status": "revoked", "refresh_hash": None} diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_keyed_oauth_flow.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_keyed_oauth_flow.py index a0fc953fb20..db4eb31775a 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_keyed_oauth_flow.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_keyed_oauth_flow.py @@ -246,7 +246,7 @@ async def test_headerless_token_exchange_retains_original_key(monkeypatch: pytes assert response.status_code == 400 stored.assert_not_awaited() if fault == "replayed": - store.revoke.assert_awaited_once_with(flow.jti) + store.revoke.assert_awaited_once_with(flow.jti, expected_status="active") @pytest.fixture @@ -775,3 +775,87 @@ async def test_cached_discovery_outlives_initiation_without_reopening_authorizat flow.resource, ) assert rejected.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_parallel_code_redemption_preserves_in_flight_winner(monkeypatch, active_grant) -> None: + import asyncio + import base64 + import hashlib + from types import SimpleNamespace + from unittest.mock import AsyncMock + from fastapi.responses import JSONResponse + from litellm.proxy._experimental.mcp_server import keyed_oauth_flow, discoverable_endpoints, mcp_server_manager + + original, row, store = active_grant + verifier: Final = "v" * 43 + binding: Final = original.model_copy( + update={ + "code_challenge": base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).rstrip(b"=").decode() + } + ) + row.binding_b64 = _seal(keyed_oauth_flow.KEYED_BINDING_PREFIX, binding) + row.status = "code" + + async def transition(grant_id, previous, following, expiry, **kwargs): + if row.status != previous: + return False + row.status = following + return True + + async def revoke(grant_id, *, expected_status=None): + if expected_status is None or row.status == expected_status: + row.status = "revoked" + + store.transition = transition + store.revoke = revoke + monkeypatch.setattr(keyed_oauth_flow, "_active_key", AsyncMock(return_value=UserAPIKeyAuth(user_id="owner"))) + server: Final = SimpleNamespace( + server_id=binding.flow.server_id, + client_id="configured-app", + is_gateway_managed_oauth2=True, + needs_user_oauth_token=True, + ) + monkeypatch.setattr(mcp_server_manager.global_mcp_server_manager, "get_mcp_server_by_id", lambda _: server) + exchanging: Final = asyncio.Event() + release: Final = asyncio.Event() + + async def upstream(**kwargs): + exchanging.set() + await release.wait() + return JSONResponse({"access_token": "vaulted-upstream-token"}) + + exchange: Final = AsyncMock(side_effect=upstream) + monkeypatch.setattr(discoverable_endpoints, "exchange_token_with_server", exchange) + code: Final = _seal( + keyed_oauth_flow.KEYED_CODE_PREFIX, + keyed_oauth_flow.KeyedCode( + grant_id=binding.flow.jti, upstream_code="upstream-code", exp=int(time.time()) + 120 + ), + ) + + async def redeem(): + return await keyed_oauth_flow.exchange_keyed_token( + _request(key=None), + "authorization_code", + code, + binding.redirect_uri, + binding.client_id, + verifier, + None, + binding.flow.resource, + None, + ) + + winner: Final = asyncio.create_task(redeem()) + try: + await asyncio.wait_for(exchanging.wait(), timeout=5) + loser: Final = await redeem() + assert loser.status_code == 400 + assert row.status == "exchanging" + finally: + release.set() + completed = await winner + assert completed.status_code == 200 + assert row.status == "active" + exchange.assert_awaited_once()