fix(mcp): recover keyed authorization and isolate concurrent code claims

This commit is contained in:
Joshua Valluru 2026-09-17 17:28:44 -07:00
parent b9512f9704
commit 31683e0e52
7 changed files with 215 additions and 8 deletions

View file

@ -543,7 +543,33 @@ class MCPRequestHandler:
from litellm.proxy._experimental.mcp_server.keyed_oauth_flow import validate_keyed_bearer
validated_user_api_key_auth = await validate_keyed_bearer(request, validated_user_api_key_auth)
try:
validated_user_api_key_auth = await validate_keyed_bearer(request, validated_user_api_key_auth)
except HTTPException as exc:
if (
exc.status_code == 401
and validated_user_api_key_auth.via_virtual_key
and request_route.rstrip("/") == "/mcp"
):
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
from litellm.proxy._experimental.mcp_server.server import (
_raise_preemptive_401_for_unauthenticated_servers, # pyright: ignore[reportPrivateUsage] # reuse the authorized MCP discovery challenge
)
allowed_for_recovery: Final = await global_mcp_server_manager.get_allowed_mcp_servers(
validated_user_api_key_auth
)
await _raise_preemptive_401_for_unauthenticated_servers(
scope=scope,
mcp_servers=mcp_servers,
oauth2_headers=None,
mcp_server_auth_headers=None,
user_api_key_auth=validated_user_api_key_auth,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
allowed_server_ids=frozenset(allowed_for_recovery),
force_keyed_authorization=True,
)
raise
# Leak-defense (single chokepoint): a gateway admission credential (session bearer or bridge
# envelope) is NEVER a valid upstream token. Scrub it from EVERY egress context so no

View file

@ -2163,8 +2163,11 @@ class KeyedOAuthGrantStore:
}
return await self._table.update_many(where=where, data=data) == 1
async def revoke(self, grant_id: str) -> None:
where: Final[prisma_db_types.LiteLLM_MCPKeyedOAuthGrantWhereInput] = {"id": grant_id}
async def revoke(self, grant_id: str, *, expected_status: KeyedGrantStatus | None = None) -> None:
where: Final[prisma_db_types.LiteLLM_MCPKeyedOAuthGrantWhereInput] = {
"id": grant_id,
**({"status": expected_status} if expected_status else {}),
}
data: Final[prisma_db_types.LiteLLM_MCPKeyedOAuthGrantUpdateManyMutationInput] = {
"status": "revoked",
"refresh_hash": None,

View file

@ -433,7 +433,7 @@ async def exchange_keyed_token(
if not await store.transition(
grant_id, "code", "exchanging", _expires(120), expected_hash=hash_token(code or "")
):
await store.revoke(grant_id)
await store.revoke(grant_id, expected_status="active")
return _oauth_error(400, "invalid_grant", "Authorization code already used or expired")
upstream: Final = await exchange_token_with_server(
request=request,

View file

@ -13,7 +13,7 @@ import time
import traceback
import types
import uuid
from collections.abc import AsyncIterator, Callable, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Collection, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol
from urllib.parse import urlencode
@ -4021,8 +4021,9 @@ if MCP_AVAILABLE:
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
user_api_key_auth: UserAPIKeyAuth | None,
client_ip: str | None,
allowed_server_ids: set[str] | None = None,
allowed_server_ids: Collection[str] | None = None,
raw_headers: Mapping[str, str] | None = None,
force_keyed_authorization: bool = False,
) -> None:
"""Fail fast with HTTP 401 for MCP servers that need user auth but
didn't receive it on this request. Covers both gateway-managed OAuth2
@ -4077,7 +4078,9 @@ if MCP_AVAILABLE:
# authorization server is the gateway itself, vaulting via the
# authorize interlude); the per-server relay advertised below
# cannot vault without a litellm key on its token request.
if await global_mcp_server_manager.has_user_oauth_token(server, user_api_key_auth):
if not force_keyed_authorization and await global_mcp_server_manager.has_user_oauth_token(
server, user_api_key_auth
):
continue
if _is_mcp_admitted_user_subject(user_api_key_auth):

View file

@ -9431,3 +9431,83 @@ class TestScopedSessionAdmission:
def test_scope_field_cannot_be_forged_through_construction(self):
forged = UserAPIKeyAuth(user_id="u1", mcp_session_resource_server_id="any-server")
assert forged.mcp_session_resource_server_id is None
@pytest.mark.asyncio
@pytest.mark.parametrize("fault", ("revoked", "expired", "malformed", "database", "denied"))
async def test_rejected_keyed_bearer_restarts_original_key_flow_without_sso(monkeypatch, fault):
import time
from types import SimpleNamespace
from urllib.parse import parse_qs, urlsplit
from starlette.requests import Request
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow, mcp_server_manager
from litellm.proxy._experimental.mcp_server.auth import user_api_key_auth_mcp
from litellm.types.mcp_server.mcp_server_manager import MCPServer
monkeypatch.setenv("LITELLM_SALT_KEY", "keyed-recovery-test-salt")
monkeypatch.setenv("PROXY_BASE_URL", "http://localhost:4000")
server = MCPServer(
server_id="recovery-server",
alias="interactive",
name="interactive",
transport="http",
auth_type="oauth2",
oauth2_flow="authorization_code",
client_id="configured-app",
authorization_url="https://upstream.example/authorize",
token_url="https://upstream.example/token",
)
manager = mcp_server_manager.global_mcp_server_manager
monkeypatch.setitem(manager.registry, server.server_id, server)
allowed = AsyncMock(return_value=[] if fault == "denied" else [server.server_id])
monkeypatch.setattr(manager, "get_allowed_mcp_servers", allowed)
monkeypatch.setattr(manager, "has_user_oauth_token", AsyncMock(return_value=True))
original = UserAPIKeyAuth(user_id="owner", models=["restricted-model"])
original.via_virtual_key = True
monkeypatch.setattr(user_api_key_auth_mcp, "user_api_key_auth", AsyncMock(return_value=original))
store = SimpleNamespace(
get=AsyncMock(
return_value=SimpleNamespace(status="revoked"),
side_effect=RuntimeError("private-db-detail") if fault == "database" else None,
)
)
monkeypatch.setattr(keyed_oauth_flow, "_store", lambda: store)
token = keyed_oauth_flow._seal(
keyed_oauth_flow.KEYED_ACCESS_PREFIX,
keyed_oauth_flow.KeyedToken(
kind="access", grant_id="grant", jti="access", exp=0 if fault == "expired" else int(time.time()) + 600
),
)
if fault == "malformed":
token = "llm_kaccess_invalid"
scope = {
"type": "http",
"method": "POST",
"scheme": "http",
"path": "/mcp",
"headers": [
(b"host", b"localhost:4000"),
(b"x-litellm-api-key", b"Bearer sk-owned"),
(b"x-mcp-servers", b"interactive"),
(b"authorization", f"Bearer {token}".encode()),
],
}
with pytest.raises(HTTPException) as rejected:
await MCPRequestHandler.process_mcp_request(scope)
if fault == "database":
assert rejected.value.status_code == 503
assert "private-db-detail" not in str(rejected.value.detail)
allowed.assert_not_awaited()
elif fault == "denied":
assert rejected.value.status_code == 401
assert not rejected.value.headers
else:
assert rejected.value.status_code == 401
assert rejected.value.headers and "www-authenticate" in rejected.value.headers
metadata = rejected.value.headers["www-authenticate"].split('resource_metadata="')[1].rstrip('"')
query = parse_qs(urlsplit(metadata).query)
assert query["mcp_server_name"] == ["interactive"]
opened = keyed_oauth_flow.open_keyed_oauth_flow(Request(scope), query["flow"][0])
assert opened.user_id == "owner" and opened.server_id == server.server_id
assert allowed.call_args.args[0] is original
assert original.models == ["restricted-model"]

View file

@ -1802,3 +1802,14 @@ async def test_keyed_grant_never_trusts_stale_replica_after_revocation(writer_do
else:
assert await store.get("revoked-grant") is authoritative
reader_table.find_unique.assert_not_awaited()
@pytest.mark.asyncio
async def test_code_replay_revokes_only_previously_issued_tokens() -> None:
from litellm.proxy._experimental.mcp_server.db import KeyedOAuthGrantStore
table: Final = SimpleNamespace(update_many=AsyncMock(return_value=0))
store: Final = KeyedOAuthGrantStore(SimpleNamespace(db=SimpleNamespace(litellm_mcpkeyedoauthgrant=table)))
await store.revoke("grant", expected_status="active")
assert table.update_many.call_args.kwargs["where"] == {"id": "grant", "status": "active"}
assert table.update_many.call_args.kwargs["data"] == {"status": "revoked", "refresh_hash": None}

View file

@ -246,7 +246,7 @@ async def test_headerless_token_exchange_retains_original_key(monkeypatch: pytes
assert response.status_code == 400
stored.assert_not_awaited()
if fault == "replayed":
store.revoke.assert_awaited_once_with(flow.jti)
store.revoke.assert_awaited_once_with(flow.jti, expected_status="active")
@pytest.fixture
@ -775,3 +775,87 @@ async def test_cached_discovery_outlives_initiation_without_reopening_authorizat
flow.resource,
)
assert rejected.value.status_code == 400
@pytest.mark.asyncio
async def test_parallel_code_redemption_preserves_in_flight_winner(monkeypatch, active_grant) -> None:
import asyncio
import base64
import hashlib
from types import SimpleNamespace
from unittest.mock import AsyncMock
from fastapi.responses import JSONResponse
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow, discoverable_endpoints, mcp_server_manager
original, row, store = active_grant
verifier: Final = "v" * 43
binding: Final = original.model_copy(
update={
"code_challenge": base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).rstrip(b"=").decode()
}
)
row.binding_b64 = _seal(keyed_oauth_flow.KEYED_BINDING_PREFIX, binding)
row.status = "code"
async def transition(grant_id, previous, following, expiry, **kwargs):
if row.status != previous:
return False
row.status = following
return True
async def revoke(grant_id, *, expected_status=None):
if expected_status is None or row.status == expected_status:
row.status = "revoked"
store.transition = transition
store.revoke = revoke
monkeypatch.setattr(keyed_oauth_flow, "_active_key", AsyncMock(return_value=UserAPIKeyAuth(user_id="owner")))
server: Final = SimpleNamespace(
server_id=binding.flow.server_id,
client_id="configured-app",
is_gateway_managed_oauth2=True,
needs_user_oauth_token=True,
)
monkeypatch.setattr(mcp_server_manager.global_mcp_server_manager, "get_mcp_server_by_id", lambda _: server)
exchanging: Final = asyncio.Event()
release: Final = asyncio.Event()
async def upstream(**kwargs):
exchanging.set()
await release.wait()
return JSONResponse({"access_token": "vaulted-upstream-token"})
exchange: Final = AsyncMock(side_effect=upstream)
monkeypatch.setattr(discoverable_endpoints, "exchange_token_with_server", exchange)
code: Final = _seal(
keyed_oauth_flow.KEYED_CODE_PREFIX,
keyed_oauth_flow.KeyedCode(
grant_id=binding.flow.jti, upstream_code="upstream-code", exp=int(time.time()) + 120
),
)
async def redeem():
return await keyed_oauth_flow.exchange_keyed_token(
_request(key=None),
"authorization_code",
code,
binding.redirect_uri,
binding.client_id,
verifier,
None,
binding.flow.resource,
None,
)
winner: Final = asyncio.create_task(redeem())
try:
await asyncio.wait_for(exchanging.wait(), timeout=5)
loser: Final = await redeem()
assert loser.status_code == 400
assert row.status == "exchanging"
finally:
release.set()
completed = await winner
assert completed.status_code == 200
assert row.status == "active"
exchange.assert_awaited_once()