From 3060361b0992941134a1d0519ee8888dd7702664 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 20 Jul 2026 17:42:34 -0700 Subject: [PATCH] test(e2e): expand MCP coverage registry to the full gateway surface The mcp.yaml denominator only enumerated the six JSON-RPC protocol operations. Studying the MCP module and its swagger surface (litellm/proxy/_experimental/ mcp_server/) surfaces a much larger customer-noticeable surface: server management CRUD + health, the non-admin submit -> admin approve governance workflow, the OAuth BYOK Authorize & Fetch Token flow with PKCE, per-user credentials and env vars, tool discovery and semantic tool search with its permission gate, toolsets, and the gateway-managed upstream OAuth leg. Adds 31 cells grounded in real file:line, keeping auth_family the client-to-gateway axis (a virtual key injecting an upstream bearer stays api_key; oauth is the BYOK/upstream authorize+token flow; none is the public surfaces) and extending the mcp assertion vocab (admin_only, persists, rejects_bad_pkce, ...). MCP server management stays in the mcp module rather than mgmt since it is MCP-domain. The grammar block in tests/e2e/CLAUDE.md is updated to match. These are denominator rows; covering tests land in follow-up PRs. --- tests/e2e/CLAUDE.md | 22 ++- tests/e2e/coverage_registry/mcp.yaml | 262 +++++++++++++++++++++++++++ 2 files changed, 282 insertions(+), 2 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 47f3c74d7f1..1815778438c 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -117,10 +117,28 @@ MCPs - endpoint features with the protocol op as the variant ``` mcp... - operation : list_tools | call_tool | list_resources | read_resource | list_prompts | get_prompt + operation : protocol ops list_tools | call_tool | list_resources | read_resource + | list_resource_templates | list_prompts | get_prompt + server mgmt server_add | server_list | server_get | server_update + | server_delete | server_health + governance server_submit | server_approve | server_reject | make_public + oauth (byok) oauth_authorize | oauth_token | oauth_metadata + oauth (upstream) oauth_session | upstream_authorize + per-user user_credential_set | user_credential_delete + | oauth_user_credential_status | user_env_vars_set + discovery tools_rest | tool_search | access_groups | discover | registry_json + toolsets toolset_add | toolset_list | toolset_delete auth_family : none | api_key | bearer | oauth - assertion : succeeds | denied_without_permission + (client-to-gateway auth, NOT the upstream's auth_type: a virtual key + injecting an upstream bearer is still api_key; oauth is the BYOK/upstream + authorize+token flow; none is public/anonymous surfaces) + assertion : succeeds | denied_without_permission | admin_only | persists | reports_status + | creates_pending | issues_code | exchanges_code | rejects_bad_pkce + | rejects_reused_code | discoverable | scoped | public_filtered + | rejects_undeclared | redirects_upstream e.g. mcp.call_tool.oauth.succeeds + mcp.oauth_token.oauth.rejects_bad_pkce + mcp.server_add.api_key.admin_only ``` Reliability & Performance - behavior features (no route; endpoint is exercised_on) diff --git a/tests/e2e/coverage_registry/mcp.yaml b/tests/e2e/coverage_registry/mcp.yaml index d477b257cb0..4a10809434c 100644 --- a/tests/e2e/coverage_registry/mcp.yaml +++ b/tests/e2e/coverage_registry/mcp.yaml @@ -111,3 +111,265 @@ assertions: [succeeds] source: "server.py:1089" rationale: Smoke; rarely used; same auth model as tools +- id: mcp.list_resource_templates.api_key.succeeds + module: mcp + tier: P2 + operation: list_resource_templates + auth_family: api_key + assertions: [succeeds] + source: "server.py:1196" + rationale: Resource-template listing; same auth stack as tools + +# Server management REST (/v1/mcp/server). Grounded in mcp_management_endpoints.py. +- id: mcp.server_add.api_key.admin_only + module: mcp + tier: P0 + operation: server_add + auth_family: api_key + assertions: [admin_only, persists] + source: "mcp_management_endpoints.py:1345" + rationale: PROXY_ADMIN-gated server registration; persists to DB and is picked up without restart; onboarding path +- id: mcp.server_list.api_key.succeeds + module: mcp + tier: P1 + operation: server_list + auth_family: api_key + assertions: [succeeds] + source: "mcp_management_endpoints.py:895" + rationale: List registered servers; admin required only to query another team +- id: mcp.server_get.api_key.succeeds + module: mcp + tier: P2 + operation: server_get + auth_family: api_key + assertions: [succeeds] + source: "mcp_management_endpoints.py:1245" + rationale: Fetch one server; non-admins filtered to their allowed servers +- id: mcp.server_update.api_key.admin_only + module: mcp + tier: P1 + operation: server_update + auth_family: api_key + assertions: [admin_only, persists] + source: "mcp_management_endpoints.py:2282" + rationale: PROXY_ADMIN-gated edit persists to DB +- id: mcp.server_delete.api_key.admin_only + module: mcp + tier: P1 + operation: server_delete + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:1758" + rationale: PROXY_ADMIN-gated delete removes the server row +- id: mcp.server_health.api_key.reports_status + module: mcp + tier: P1 + operation: server_health + auth_family: api_key + assertions: [reports_status] + source: "mcp_management_endpoints.py:999" + rationale: Health check distinguishes a reachable upstream from a broken one + +# Governance / submission workflow (non-admin submit -> admin review). +- id: mcp.server_submit.api_key.creates_pending + module: mcp + tier: P1 + operation: server_submit + auth_family: api_key + assertions: [creates_pending] + source: "mcp_management_endpoints.py:1050" + rationale: Non-admin submission creates a pending row for review; admins auto-approve +- id: mcp.server_approve.api_key.admin_only + module: mcp + tier: P1 + operation: server_approve + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:1154" + rationale: PROXY_ADMIN approves a pending submission into an active server +- id: mcp.server_reject.api_key.admin_only + module: mcp + tier: P2 + operation: server_reject + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:1198" + rationale: PROXY_ADMIN rejects a pending submission +- id: mcp.make_public.api_key.admin_only + module: mcp + tier: P2 + operation: make_public + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:2411" + rationale: PROXY_ADMIN marks servers public for the AI Hub + +# OAuth BYOK "Authorize & Fetch Token" flow (byok_oauth_endpoints.py, include_in_schema=False). +- id: mcp.oauth_authorize.oauth.issues_code + module: mcp + tier: P1 + operation: oauth_authorize + auth_family: oauth + assertions: [issues_code] + source: "byok_oauth_endpoints.py:699" + rationale: Authenticated authorize stores a one-time code bound to user_id + PKCE challenge and redirects with code+state +- id: mcp.oauth_token.oauth.exchanges_code + module: mcp + tier: P1 + operation: oauth_token + auth_family: oauth + assertions: [exchanges_code] + source: "byok_oauth_endpoints.py:762" + rationale: Valid code + PKCE verifier mints a byok_session JWT and persists the per-user upstream credential +- id: mcp.oauth_token.oauth.rejects_bad_pkce + module: mcp + tier: P0 + operation: oauth_token + auth_family: oauth + assertions: [rejects_bad_pkce] + source: "byok_oauth_endpoints.py:794" + rationale: S256 verifier mismatch returns invalid_grant; the PKCE trust boundary must not be bypassable +- id: mcp.oauth_token.oauth.rejects_reused_code + module: mcp + tier: P1 + operation: oauth_token + auth_family: oauth + assertions: [rejects_reused_code] + source: "byok_oauth_endpoints.py:832" + rationale: Authorization code is one-time-use; a replay is refused +- id: mcp.oauth_metadata.none.discoverable + module: mcp + tier: P2 + operation: oauth_metadata + auth_family: none + assertions: [discoverable] + source: "byok_oauth_endpoints.py:599" + rationale: Public .well-known AS + protected-resource metadata advertise S256 + authorization_code so MCP clients discover the flow + +# Per-user credentials + env vars (caller manages its own). +- id: mcp.user_credential_set.api_key.persists + module: mcp + tier: P1 + operation: user_credential_set + auth_family: api_key + assertions: [persists] + source: "mcp_management_endpoints.py:1824" + rationale: Caller stores its own BYOK key for a server; injected at egress on later tool calls +- id: mcp.user_credential_delete.api_key.succeeds + module: mcp + tier: P2 + operation: user_credential_delete + auth_family: api_key + assertions: [succeeds] + source: "mcp_management_endpoints.py:1861" + rationale: Caller revokes its own stored BYOK key +- id: mcp.oauth_user_credential_status.api_key.succeeds + module: mcp + tier: P2 + operation: oauth_user_credential_status + auth_family: api_key + assertions: [succeeds] + source: "mcp_management_endpoints.py:1979" + rationale: Caller checks whether it has a stored upstream OAuth2 credential +- id: mcp.user_env_vars_set.api_key.rejects_undeclared + module: mcp + tier: P2 + operation: user_env_vars_set + auth_family: api_key + assertions: [persists, rejects_undeclared] + source: "mcp_management_endpoints.py:2185" + rationale: Per-user env vars persist, but only admin-declared var names are accepted + +# Tool discovery / search. +- id: mcp.tools_rest.api_key.succeeds + module: mcp + tier: P1 + operation: tools_rest + auth_family: api_key + assertions: [succeeds] + source: "mcp_management_endpoints.py:713" + rationale: Flat REST tool listing scoped to the calling key's allowed servers +- id: mcp.tool_search.api_key.denied_without_permission + module: mcp + tier: P1 + operation: tool_search + auth_family: api_key + assertions: [denied_without_permission] + source: "server.py:851-864" + rationale: Semantic tool-search virtual tool is gated by object_permission.mcp_tool_search_enabled +- id: mcp.tool_search.api_key.succeeds + module: mcp + tier: P2 + operation: tool_search + auth_family: api_key + assertions: [succeeds] + source: "tool_search.py:116" + rationale: Semantic tool search returns ranked tools for a permitted key +- id: mcp.access_groups.api_key.succeeds + module: mcp + tier: P2 + operation: access_groups + auth_family: api_key + assertions: [succeeds] + source: "mcp_management_endpoints.py:737" + rationale: List MCP access groups the key can see +- id: mcp.discover.api_key.admin_only + module: mcp + tier: P2 + operation: discover + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:2505" + rationale: PROXY_ADMIN-only curated well-known server discovery for the UI +- id: mcp.registry_json.none.public_filtered + module: mcp + tier: P2 + operation: registry_json + auth_family: none + assertions: [public_filtered] + source: "mcp_management_endpoints.py:786" + rationale: Public registry is gated by a feature flag and IP-filtered to public servers for external callers + +# Toolsets (named bundles of tools). +- id: mcp.toolset_add.api_key.admin_only + module: mcp + tier: P2 + operation: toolset_add + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:2610" + rationale: PROXY_ADMIN creates a toolset +- id: mcp.toolset_list.api_key.scoped + module: mcp + tier: P2 + operation: toolset_list + auth_family: api_key + assertions: [scoped] + source: "mcp_management_endpoints.py:2650" + rationale: Toolset listing is filtered to the key's object_permission.mcp_toolsets +- id: mcp.toolset_delete.api_key.admin_only + module: mcp + tier: P2 + operation: toolset_delete + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:2749" + rationale: PROXY_ADMIN deletes a toolset + +# Gateway-managed upstream OAuth (delegate-auth / per-user token; see the LIT-4627 discussion). +- id: mcp.oauth_session.api_key.admin_only + module: mcp + tier: P2 + operation: oauth_session + auth_family: api_key + assertions: [admin_only] + source: "mcp_management_endpoints.py:1436" + rationale: PROXY_ADMIN caches a temp in-memory server for the OAuth setup flow (no DB write) +- id: mcp.upstream_authorize.oauth.redirects_upstream + module: mcp + tier: P2 + operation: upstream_authorize + auth_family: oauth + assertions: [redirects_upstream] + source: "mcp_management_endpoints.py:1644" + rationale: Gateway-managed authorization_code begins the upstream OAuth authorize for a server; the delegate-auth leg needs a real OAuth upstream to prove end to end