fix(mcp): enforce authoritative keyed grant reads and safe failures

This commit is contained in:
Joshua Valluru 2026-09-17 17:09:04 -07:00
parent 93596e036c
commit b9512f9704
4 changed files with 303 additions and 20 deletions

View file

@ -2099,7 +2099,9 @@ KeyedGrantStatus = Literal["pending", "authorizing", "code", "exchanging", "acti
class KeyedOAuthGrantStore:
def __init__(self, prisma_client: PrismaClient) -> None:
self._table = MCPKeyedOAuthGrantRepository(prisma_client).table
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient, writer_wrapper
self._table = MCPKeyedOAuthGrantRepository(WriterPinnedClient(writer_wrapper(prisma_client.db))).table
async def begin(self, grant_id: str, binding: str, expires_at: datetime) -> None:
now: Final = datetime.now(timezone.utc)

View file

@ -76,7 +76,7 @@ def open_keyed_oauth_flow(request: Request, value: str, *, allow_expired: bool =
def keyed_resource_metadata(request: Request, value: str) -> JSONResponse:
flow: Final = open_keyed_oauth_flow(request, value)
flow: Final = open_keyed_oauth_flow(request, value, allow_expired=True)
metadata: Final = ProtectedResourceMetadata.model_validate(
MappingProxyType(
{
@ -139,6 +139,10 @@ async def keyed_authorize(
raise HTTPException(status_code=400, detail="OAuth resource does not match this connection")
if not re.fullmatch(r"[A-Za-z0-9_-]{43}", code_challenge or ""):
raise HTTPException(status_code=400, detail="A valid S256 PKCE challenge is required")
base: Final = get_request_base_url(request)
parsed: Final = urlparse(base)
if parsed.scheme != "https" and parsed.hostname not in ("localhost", "127.0.0.1", "::1"):
raise HTTPException(status_code=400, detail="Key confirmation requires HTTPS outside localhost")
binding: Final = KeyedAuthorization(
flow=flow, client_id=client_id, redirect_uri=redirect_uri, state=state, code_challenge=code_challenge or ""
)
@ -147,10 +151,6 @@ async def keyed_authorize(
if stored != binding:
raise HTTPException(status_code=400, detail="This connection has already started. Reconnect your MCP client.")
browser: Final = KeyedBrowser(grant_id=flow.jti, csrf=secrets.token_urlsafe(32), exp=flow.exp)
base: Final = get_request_base_url(request)
parsed: Final = urlparse(base)
if parsed.scheme != "https" and parsed.hostname not in ("localhost", "127.0.0.1", "::1"):
raise HTTPException(status_code=400, detail="Key confirmation requires HTTPS outside localhost")
response: Final = HTMLResponse(
"<!doctype html><html><head><title>Connect MCP server</title></head><body>"
"<h1>Confirm your LiteLLM key</h1>"
@ -237,8 +237,13 @@ def _expires(seconds: int) -> datetime:
return datetime.fromtimestamp(time.time() + seconds, timezone.utc)
async def _binding(grant_id: str, request: Request) -> KeyedAuthorization:
row: Final = await _store().get(grant_id)
async def _binding(grant_id: str, request: Request, *, require_active: bool = False) -> KeyedAuthorization:
try:
row: Final = await _store().get(grant_id)
except Exception as exc:
raise HTTPException(status_code=503, detail="OAuth authorization storage is unavailable") from exc
if require_active and (row is None or row.status != "active"):
raise HTTPException(status_code=401, detail="MCP authorization grant is revoked")
if row is None or row.status == "revoked" or row.expires_at <= datetime.now(timezone.utc):
raise HTTPException(status_code=400, detail="OAuth grant is expired or revoked. Reconnect your client.")
binding: Final = _open_sealed(row.binding_b64, KEYED_BINDING_PREFIX, KeyedAuthorization, "keyed_binding")
@ -292,8 +297,6 @@ async def confirm_keyed_authorization(request: Request) -> Response:
):
raise HTTPException(status_code=403, detail="Key confirmation expired. Reconnect your client.")
binding: Final = await _binding(grant_id, request)
if binding.flow.resource != f"{base}/mcp":
raise HTTPException(status_code=403, detail="Key confirmation belongs to another gateway")
store: Final = _store()
if not await store.attempt(grant_id):
raise HTTPException(status_code=403, detail="Key confirmation expired. Reconnect your client.")
@ -479,7 +482,12 @@ async def validate_keyed_bearer(request: Request, auth: UserAPIKeyAuth) -> UserA
token: Final = _open_sealed(value, KEYED_ACCESS_PREFIX, KeyedToken, "keyed_access")
if token is None or token.kind != "access" or token.exp <= time.time():
raise HTTPException(status_code=401, detail="MCP authorization token is invalid or expired")
binding: Final = await _binding(token.grant_id, request)
try:
binding: Final = await _binding(token.grant_id, request, require_active=True)
except HTTPException as exc:
if exc.status_code == 400:
raise HTTPException(status_code=401, detail="MCP authorization grant is invalid or expired") from exc
raise
presented_key: Final = _litellm_key_from_request(request)
if (
not auth.via_virtual_key
@ -488,9 +496,6 @@ async def validate_keyed_bearer(request: Request, auth: UserAPIKeyAuth) -> UserA
or not hmac.compare_digest(hash_token(presented_key), binding.flow.key_hash)
):
raise HTTPException(status_code=403, detail="MCP authorization requires its original virtual key")
row: Final = await _store().get(token.grant_id)
if row is None or row.status != "active":
raise HTTPException(status_code=401, detail="MCP authorization grant is revoked")
return auth.model_copy(update=MappingProxyType({"mcp_session_resource_server_id": binding.flow.server_id}))

View file

@ -1773,3 +1773,32 @@ async def test_keyed_grant_begin_preserves_existing_binding_and_rechecks_cleanup
table.delete_many.assert_not_awaited()
assert await store.get("new-grant") is row
assert table.find_unique.call_args.kwargs["where"] == {"id": "new-grant"}
@pytest.mark.asyncio
@pytest.mark.parametrize("writer_down", (False, True))
async def test_keyed_grant_never_trusts_stale_replica_after_revocation(writer_down: bool) -> None:
from litellm.proxy._experimental.mcp_server.db import KeyedOAuthGrantStore
from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
authoritative: Final = SimpleNamespace(status="revoked")
writer_table: Final = SimpleNamespace(
find_many=AsyncMock(),
create=AsyncMock(),
find_unique=AsyncMock(
return_value=authoritative, side_effect=RuntimeError("writer unavailable") if writer_down else None
),
)
reader_table: Final = SimpleNamespace(find_unique=AsyncMock(return_value=SimpleNamespace(status="active")))
db: Final = RoutingPrismaWrapper(
writer=SimpleNamespace(litellm_mcpkeyedoauthgrant=writer_table),
reader=SimpleNamespace(litellm_mcpkeyedoauthgrant=reader_table),
)
db._writer_unavailable = writer_down
store: Final = KeyedOAuthGrantStore(SimpleNamespace(db=db))
if writer_down:
with pytest.raises(RuntimeError, match="writer unavailable"):
await store.get("revoked-grant")
else:
assert await store.get("revoked-grant") is authoritative
reader_table.find_unique.assert_not_awaited()

View file

@ -83,7 +83,23 @@ def test_keyed_authorization_requires_an_owned_key(missing: str) -> None:
@pytest.mark.asyncio
@pytest.mark.parametrize("fault", ("none", "client", "pkce", "resource", "replayed", "storage"))
@pytest.mark.parametrize(
"fault",
(
"none",
"client",
"pkce",
"resource",
"replayed",
"storage",
"expired",
"upstream",
"server",
"name",
"named",
"claim_outage",
),
)
async def test_headerless_token_exchange_retains_original_key(monkeypatch: pytest.MonkeyPatch, fault: str) -> None:
import base64
import hashlib
@ -149,6 +165,10 @@ async def test_headerless_token_exchange_retains_original_key(monkeypatch: pytes
transition=AsyncMock(return_value=fault != "replayed"),
revoke=AsyncMock(),
)
if fault == "claim_outage":
store.transition.side_effect = RuntimeError("private-database-details")
if fault == "server":
monkeypatch.delitem(registry, server.server_id)
monkeypatch.setattr(keyed_oauth_flow, "_store", lambda: store)
original_key: Final = UserAPIKeyAuth(api_key=flow.key_hash, user_id="owner", models=["limited-model"])
monkeypatch.setattr(
@ -170,12 +190,19 @@ async def test_headerless_token_exchange_retains_original_key(monkeypatch: pytes
assert b"client_id=configured-client" in outbound.content
assert "x-litellm-api-key" not in outbound.headers
return httpx.Response(
200, json={"access_token": "upstream-token", "refresh_token": "upstream-refresh", "token_type": "Bearer"}
401 if fault == "upstream" else 200,
json={"error": "invalid_grant"}
if fault == "upstream"
else {"access_token": "upstream-token", "refresh_token": "upstream-refresh", "token_type": "Bearer"},
)
code: Final = _seal(
keyed_oauth_flow.KEYED_CODE_PREFIX,
keyed_oauth_flow.KeyedCode(grant_id=flow.jti, upstream_code="real-upstream-code", exp=int(time.time()) + 120),
keyed_oauth_flow.KeyedCode(
grant_id=flow.jti,
upstream_code="real-upstream-code",
exp=0 if fault == "expired" else int(time.time()) + 120,
),
)
async with httpx.AsyncClient(transport=httpx.MockTransport(upstream_response)) as transport:
upstream: Final = AsyncHTTPHandler()
@ -193,9 +220,9 @@ async def test_headerless_token_exchange_retains_original_key(monkeypatch: pytes
client_secret=None,
refresh_token=None,
scope=None,
mcp_server_name=None,
mcp_server_name="missing-server" if fault == "name" else server.name if fault == "named" else None,
)
if fault == "none":
if fault in ("none", "named"):
assert response.status_code == 200
result: Final = json.loads(response.body)
assert result["access_token"].startswith(keyed_oauth_flow.KEYED_ACCESS_PREFIX)
@ -206,7 +233,13 @@ async def test_headerless_token_exchange_retains_original_key(monkeypatch: pytes
assert stored.call_args.kwargs["require_persistence"] is True
assert permission.call_args.args[1] is original_key
assert permission.call_args.args[1].models == ["limited-model"]
elif fault == "storage":
elif fault == "upstream":
assert response.status_code == 400
assert json.loads(response.body)["error"] == "invalid_grant"
assert store.transition.await_count == 1
assert store.transition.call_args.args[1:3] == ("code", "exchanging")
stored.assert_not_awaited()
elif fault in ("storage", "claim_outage"):
assert response.status_code == 503
assert "access_token" not in json.loads(response.body)
else:
@ -528,3 +561,217 @@ async def test_token_storage_outage_returns_safe_oauth_error(active_grant) -> No
assert response.status_code == 503
assert json.loads(response.body)["error"] == "server_error"
assert b"private-database" not in response.body
@pytest.mark.asyncio
@pytest.mark.parametrize("fault", ("revoked", "missing", "database", "expired"))
async def test_mcp_bearer_rejection_has_safe_transport_status(active_grant, fault: str) -> None:
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow
binding, row, store = active_grant
if fault == "database":
store.get.side_effect = RuntimeError("private-database-details")
elif fault == "missing":
store.get.return_value = None
elif fault == "expired":
from datetime import datetime, timezone
row.expires_at = datetime.fromtimestamp(0, timezone.utc)
else:
row.status = "revoked"
token: Final = _seal(
keyed_oauth_flow.KEYED_ACCESS_PREFIX,
keyed_oauth_flow.KeyedToken(kind="access", grant_id=binding.flow.jti, jti="access", exp=int(time.time()) + 600),
)
request: Final = _request()
request.scope["headers"].append((b"authorization", f"Bearer {token}".encode()))
auth: Final = UserAPIKeyAuth(user_id="owner")
auth.via_virtual_key = True
with pytest.raises(HTTPException) as rejected:
await keyed_oauth_flow.validate_keyed_bearer(request, auth)
assert rejected.value.status_code == (503 if fault == "database" else 401)
assert "private-database" not in str(rejected.value.detail)
@pytest.mark.asyncio
@pytest.mark.parametrize("fault", ("policy", "resource", "pkce", "conflict", "http"))
async def test_key_confirmation_refuses_invalid_authorization_context(
monkeypatch: pytest.MonkeyPatch, fault: str
) -> None:
from unittest.mock import AsyncMock
from types import SimpleNamespace
from fastapi.responses import JSONResponse
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow, gateway_dcr_flow
request: Final = _request(host="gateway.example" if fault == "http" else "localhost:4000")
flow: Final = start_keyed_oauth_flow(request, UserAPIKeyAuth(user_id="owner"), "server-one")
policy: Final = JSONResponse({"error": "invalid_client"}, status_code=400) if fault == "policy" else None
monkeypatch.setattr(gateway_dcr_flow, "_rejected_authorize_request", lambda *args: policy)
store: Final = SimpleNamespace(begin=AsyncMock())
monkeypatch.setattr(keyed_oauth_flow, "_store", lambda: store)
binding: Final = keyed_oauth_flow.KeyedAuthorization(
flow=open_keyed_oauth_flow(request, flow),
client_id="different" if fault == "conflict" else "registered",
redirect_uri="http://localhost:8787/callback",
state="state",
code_challenge="c" * 43,
)
monkeypatch.setattr(keyed_oauth_flow, "_binding", AsyncMock(return_value=binding))
async def authorize():
return await keyed_oauth_flow.keyed_authorize(
request,
flow,
"registered",
binding.redirect_uri,
"state",
"short" if fault == "pkce" else "c" * 43,
"S256",
"code",
"https://other.example/mcp" if fault == "resource" else binding.flow.resource,
)
if fault == "policy":
assert await authorize() is policy
store.begin.assert_not_awaited()
else:
with pytest.raises(HTTPException) as rejected:
await authorize()
assert rejected.value.status_code == 400
if fault in ("resource", "pkce", "http"):
store.begin.assert_not_awaited()
@pytest.mark.asyncio
async def test_keyed_callback_preserves_existing_identity_binding_and_client_state(
monkeypatch: pytest.MonkeyPatch, active_grant
) -> None:
from http.cookies import SimpleCookie
from urllib.parse import parse_qs, urlsplit
from unittest.mock import AsyncMock
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow, discoverable_endpoints
from litellm.types.mcp_server.mcp_server_manager import MCPServer
binding, row, store = active_grant
row.status = "authorizing"
auth: Final = UserAPIKeyAuth(user_id="owner")
monkeypatch.setattr(keyed_oauth_flow, "_active_key", AsyncMock(return_value=auth))
server: Final = MCPServer(
server_id=binding.flow.server_id,
name="interactive",
transport="http",
auth_type="oauth2",
oauth2_flow="authorization_code",
client_id="configured-app",
authorization_url="https://upstream.example/authorize",
token_url="https://upstream.example/token",
oauth_identity_binding={
"mode": "enforce",
"issuer": "https://upstream.example",
"audiences": ["configured-app"],
},
)
authorized: Final = await discoverable_endpoints.authorize_with_server(
_request(key=None),
server,
server.client_id,
binding.redirect_uri,
state=binding.state,
code_challenge=binding.code_challenge,
code_challenge_method="S256",
keyed_auth=auth,
keyed_grant_id=binding.flow.jti,
)
query: Final = parse_qs(urlsplit(authorized.headers["location"]).query)
cookies: Final = SimpleCookie()
cookies.load(authorized.headers["set-cookie"])
cookie_name: Final = discoverable_endpoints._oauth_state_cookie_name(query["state"][0])
request: Final = _request(key=None)
request.scope["path"] = "/callback"
request.scope["headers"].append((b"cookie", f"{cookie_name}={cookies[cookie_name].value}".encode()))
completed: Final = await discoverable_endpoints.callback(request, code="upstream-code", state=query["state"][0])
returned: Final = parse_qs(urlsplit(completed.headers["location"]).query)
outer: Final = keyed_oauth_flow._open_sealed(
returned["code"][0], keyed_oauth_flow.KEYED_CODE_PREFIX, keyed_oauth_flow.KeyedCode, "test"
)
inner: Final = discoverable_endpoints.open_bridge_authorization_code(outer.upstream_code)
assert outer.grant_id == binding.flow.jti
assert inner.upstream_code == "upstream-code"
assert inner.litellm_user_id == binding.flow.user_id
assert inner.mcp_server_id == binding.flow.server_id
assert inner.oauth_nonce == query["nonce"][0]
assert returned["state"] == [binding.state]
assert store.transition.call_args.kwargs["code_hash"] == hash_token(returned["code"][0])
@pytest.mark.asyncio
async def test_missing_database_blocks_authorization(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy import proxy_server
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow
monkeypatch.setattr(proxy_server, "prisma_client", None)
with pytest.raises(HTTPException) as rejected:
await keyed_oauth_flow._binding("grant", _request(key=None))
assert rejected.value.status_code == 503
@pytest.mark.asyncio
@pytest.mark.parametrize("fault", ("expired", "corrupt", "other_resource"))
async def test_unusable_durable_binding_cannot_authorize(active_grant, fault: str) -> None:
from datetime import datetime, timezone
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow
_, row, _ = active_grant
if fault == "expired":
row.expires_at = datetime.fromtimestamp(0, timezone.utc)
elif fault == "corrupt":
row.binding_b64 = "invalid"
with pytest.raises(HTTPException) as rejected:
await keyed_oauth_flow._binding(
"grant", _request(host="other.example" if fault == "other_resource" else "localhost:4000")
)
assert rejected.value.status_code == 400
@pytest.mark.asyncio
async def test_non_keyed_bearer_keeps_existing_admission() -> None:
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow
request: Final = _request()
request.scope["headers"].append((b"authorization", b"Bearer existing-upstream-pat"))
auth: Final = UserAPIKeyAuth(user_id="owner")
assert await keyed_oauth_flow.validate_keyed_bearer(request, auth) is auth
@pytest.mark.asyncio
async def test_cached_discovery_outlives_initiation_without_reopening_authorization() -> None:
import json
from litellm.proxy._experimental.mcp_server import keyed_oauth_flow
request: Final = _request(key=None)
flow: Final = keyed_oauth_flow.KeyedOAuthFlow(
key_hash=hash_token("sk-owned"),
user_id="owner",
server_id="server-one",
resource="http://localhost:4000/mcp",
jti="expired-flow",
exp=1,
)
sealed: Final = _seal(KEYED_FLOW_PREFIX, flow)
resource: Final = keyed_oauth_flow.keyed_resource_metadata(request, sealed)
issuer: Final = keyed_oauth_flow.keyed_server_metadata(request, sealed)
assert json.loads(resource.body)["authorization_servers"] == [json.loads(issuer.body)["issuer"]]
assert json.loads(issuer.body)["token_endpoint"] == "http://localhost:4000/token"
with pytest.raises(HTTPException) as rejected:
await keyed_oauth_flow.keyed_authorize(
request,
sealed,
"client",
"http://localhost:8787/callback",
"state",
"c" * 43,
"S256",
"code",
flow.resource,
)
assert rejected.value.status_code == 400