diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 8484c0c7f61..7e71f2994b2 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -33,7 +33,7 @@ Every test under `tests/e2e/mcp/` must exercise the proxy against the real Datad - Prefer calling real Datadog tools that prove the product path (e.g. `search_datadog_logs` for list/call and permission denials). Seed a unique marker (`e2e-datadog-mcp-*`) in a chat completion when you need a log the tool can find; dual-read with `dd_logs` from conftest when delivery matters - Delete the MCP server (and any keys) through `resources.defer` the same way every other suite tears down - If a new MCP behavior cannot be covered with Datadog's tool surface, say so in the PR and get agreement before inventing another upstream; the default is always Datadog -- The one standing exception is `test_mcp_chat_completion_oauth_e2e.py`. Datadog authenticates with the static `DD-API-KEY` / `DD-APPLICATION-KEY` headers and exposes no authorize/token dance at all, so it cannot exercise gateway-managed OAuth or per-user token seeding in any form. That test drives a real Linear MCP server instead; it is still a real remote upstream, so the no-mock, no-fixture rule above holds unchanged +- Gateway-managed OAuth (authorization_code / per-user token vault) is not exercisable against Datadog's static-header MCP. When adding chat/responses/messages bridge e2e for that path, use a real remote OAuth MCP server (one shared real OAuth MCP for the suite), still no mocks or local fixtures ## Lay the pattern down in a class diff --git a/tests/e2e/coverage_registry/MCP_FEATURES.md b/tests/e2e/coverage_registry/MCP_FEATURES.md index d322b5d7edf..c86cbbd8949 100644 --- a/tests/e2e/coverage_registry/MCP_FEATURES.md +++ b/tests/e2e/coverage_registry/MCP_FEATURES.md @@ -38,7 +38,7 @@ toolset routes exist via `dynamic_mcp_route` / `toolset_mcp_route`. | Feature | Code | Registry cell(s) | E2E today | | --- | --- | --- | --- | -| list_tools | `server.py` handle_list_tools; REST `/tools/list` | `mcp.list_tools.{api_key,bearer,oauth,none}.succeeds` | api_key often skipped LIT-5052; oauth skipif Linear session | +| list_tools | `server.py` handle_list_tools; REST `/tools/list` | `mcp.list_tools.{api_key,bearer,oauth,none}.succeeds` | api_key often skipped LIT-5052; oauth bridge e2e TBD on a non-Linear real OAuth MCP | | call_tool | `server.py` mcp_server_tool_call; REST `/tools/call` | `mcp.call_tool.{api_key,bearer,oauth,none}.succeeds` | same | | list_tools denied without key scope | permission path | `mcp.list_tools.api_key.denied_without_permission` | yes | | call_tool denied without permission | rest/manager | `mcp.call_tool.api_key.denied_without_permission` | skipped LIT-5052 | @@ -82,7 +82,7 @@ toolset routes exist via `dynamic_mcp_route` / `toolset_mcp_route`. | --- | --- | --- | --- | | `none` | No upstream auth | generic succeeds | partial | | static (`api_key`, `bearer_token`, `basic`, `authorization`, `token`) | Inject static headers | `upstream_static_auth` | missing dedicated | -| `oauth2` + `authorization_code` | Per-user vault | oauth cells | Linear e2e; UI flaky (#15) | +| `oauth2` + `authorization_code` | Per-user vault | oauth cells | e2e TBD on a real OAuth MCP; UI flaky (#15) | | `oauth2` + `client_credentials` | M2M | `upstream_oauth2_client_credentials` | missing | | `delegate_auth_to_upstream` | Client PKCE with upstream | `delegate_auth_upstream` | missing | | `oauth_passthrough` | Proxy metadata + 401 challenges | `oauth_passthrough` | unit | @@ -164,7 +164,7 @@ All three share `LiteLLM_Proxy_MCP_Handler` for `litellm_proxy` / `litellm_proxy | Surface | Entry | Registry | E2E | | --- | --- | --- | --- | -| **Chat completions** `/v1/chat/completions` | `acompletion_with_mcp` via `main.py` | `mcp.chat_completion.{api_key,oauth}.auto_executes_tools` | oauth Linear (skipif); api_key missing | +| **Chat completions** `/v1/chat/completions` | `acompletion_with_mcp` via `main.py` | `mcp.chat_completion.{api_key,oauth}.auto_executes_tools` | e2e TBD on a shared real MCP for api_key and oauth | | **Responses** `/v1/responses` | `aresponses_api_with_mcp` | `mcp.responses.api_key.auto_executes_tools` | missing | | **Messages** `/v1/messages` | `anthropic_messages_with_mcp` via experimental pass-through handler | `mcp.messages.{api_key,oauth}.auto_executes_tools` | **was missing from map; no e2e** | | Semantic tool filter on chat tools | `semantic_tool_filter` + hook | `mcp.chat_completion.api_key.semantic_filter_narrows` | missing | @@ -296,6 +296,6 @@ Collector after registry expansion: **MCPs ~2/N live** until tests land; denomin ## Related docs -- E2E MCP suite: `tests/e2e/CLAUDE.md` (real Datadog; Linear for OAuth) +- E2E MCP suite: `tests/e2e/CLAUDE.md` (real Datadog for api_key; separate real OAuth MCP for vault/oauth bridges) - Registry: `tests/e2e/coverage_registry/mcp.yaml` - MCP internal note: `litellm/proxy/_experimental/mcp_server/CLAUDE.md` diff --git a/tests/e2e/coverage_registry/mcp.yaml b/tests/e2e/coverage_registry/mcp.yaml index f893a51fd0a..75a46623908 100644 --- a/tests/e2e/coverage_registry/mcp.yaml +++ b/tests/e2e/coverage_registry/mcp.yaml @@ -437,8 +437,8 @@ operation: chat_completion auth_family: oauth assertions: [auto_executes_tools] - source: "test_mcp_chat_completion_oauth_e2e.py" - rationale: "Gateway-managed OAuth: chat completion lists and executes tools with vaulted user token" + source: "responses/mcp/chat_completions_handler.py:acompletion_with_mcp" + rationale: "Gateway-managed OAuth: chat completion lists and executes tools with vaulted user token (e2e TBD on a real OAuth MCP)" - id: mcp.chat_completion.api_key.semantic_filter_narrows module: mcp tier: P1 diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 34770bc596b..408af5f7a35 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -38,8 +38,6 @@ UI_BASE_URL = os.environ.get("E2E_UI_BASE_URL", PROXY_BASE_URL).rstrip("/") CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5") CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") -LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp") -LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "") # Jaeger query API of the compose stack's OTEL trace destination (the `jaeger` # service in docker-compose.yml maps it to host 16686). Trace-completeness tests diff --git a/tests/e2e/mcp/linear_session_capture.py b/tests/e2e/mcp/linear_session_capture.py deleted file mode 100644 index 1c867e17e22..00000000000 --- a/tests/e2e/mcp/linear_session_capture.py +++ /dev/null @@ -1,58 +0,0 @@ -"""One-time helper to capture a logged-in Linear browser session for the -real-Linear MCP e2e test. - -The real-Linear test drives the genuine gateway-managed authorization_code -dance against ``mcp.linear.app``. The only step that cannot be scripted is -Linear's login (magic link / SSO), so a human authenticates once here and the -resulting session (cookies + local storage) is persisted to disk. The e2e test -then loads that session in a headless Playwright context and clicks Approve on -Linear's consent screen every run, with no human and no login automation. - -Run it with the e2e venv, log into Linear in the window that opens, then return -to the terminal and press Enter: - - LITELLM=~/litellm-mcpe2e - "$LITELLM"/.venv/bin/python "$LITELLM"/tests/e2e/mcp/linear_session_capture.py - -The session is written to ``E2E_LINEAR_STORAGE_STATE`` (default -``~/.litellm-e2e/linear_storage_state.json``), outside the repo. It is a -secret: never commit it. Re-run this whenever Linear expires the session. -""" - -from __future__ import annotations - -import os -from pathlib import Path - -from playwright.sync_api import sync_playwright - -DEFAULT_STATE_PATH = Path.home() / ".litellm-e2e" / "linear_storage_state.json" - - -def capture(state_path: Path) -> None: - """Open a headed browser at Linear, wait for the human to log in, then save - the authenticated session to ``state_path``.""" - state_path.parent.mkdir(parents=True, exist_ok=True) - with sync_playwright() as playwright: - browser = playwright.chromium.launch(headless=False) - context = browser.new_context() - page = context.new_page() - page.goto("https://linear.app/login", wait_until="domcontentloaded") - print("\n" + "=" * 72) - print("Log into Linear in the browser window that just opened.") - print("If Linear emails you a magic link, paste the link into THIS window's") - print("address bar (opening it in your default browser won't capture the") - print("session). Google SSO works too as long as you complete it here.") - print("When your Linear workspace has loaded, come back and press Enter.") - print("=" * 72) - input("Press Enter once you are logged in... ") - page.goto("https://mcp.linear.app/", wait_until="domcontentloaded") - context.storage_state(path=str(state_path)) - browser.close() - print(f"\nSaved Linear session to {state_path}") - print("Point the e2e test at it with:") - print(f' export E2E_LINEAR_STORAGE_STATE="{state_path}"') - - -if __name__ == "__main__": - capture(Path(os.environ.get("E2E_LINEAR_STORAGE_STATE", str(DEFAULT_STATE_PATH)))) diff --git a/tests/e2e/mcp/oauth_chat_client.py b/tests/e2e/mcp/oauth_chat_client.py index 2eaf512cfa5..4378a5869d0 100644 --- a/tests/e2e/mcp/oauth_chat_client.py +++ b/tests/e2e/mcp/oauth_chat_client.py @@ -3,7 +3,7 @@ Registers a gateway-managed OAuth (authorization_code) MCP server, seeds the per-user upstream token by driving the interactive authorize dance with the official mcp SDK's OAuthClientProvider (the browser leg is a headless Chromium -primed with a human's saved Linear session), then exercises the server through +primed with a human's saved browser session for the OAuth MCP under test), then exercises the server through /chat/completions, where the gateway lists and executes its tools with the stored per-user token. @@ -71,13 +71,12 @@ class InMemoryTokenStorage: async def _browser_follow_authorize(start_url: str, storage_state_path: str) -> tuple[str, str | None]: """Play the browser's role for a real upstream whose authorize endpoint - serves an interactive consent page (Linear). A headless Chromium primed - with a human's saved Linear session opens the gateway authorize URL and - clicks through Linear's consent screens (the mcp.linear.app Approve form, - then the linear.app workspace-selection page), riding the rest of the chain - (Linear -> gateway callback -> host redirect_uri). The final hop is - intercepted and short-circuited, since nothing listens there, and its - code/state are read off the query string.""" + serves an interactive consent page. A headless Chromium primed with a saved + browser session opens the gateway authorize URL, advances common consent + controls (Approve / Authorize / Allow), and rides the chain through the + gateway callback to the host redirect_uri. The final hop is intercepted and + short-circuited, since nothing listens there, and its code/state are read off + the query string.""" from playwright.async_api import async_playwright captured: dict[str, str] = {} # mutable-ok: hand-off from the request listener diff --git a/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py b/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py deleted file mode 100644 index 4447893af3c..00000000000 --- a/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py +++ /dev/null @@ -1,203 +0,0 @@ -"""On-demand e2e: a chat completion drives a gateway-managed OAuth MCP server. - -The real end-user flow for MCP over an OAuth server: a user registers a Linear -authorization_code server, authorizes it once so the gateway stores their -upstream token, then sends a normal /chat/completions request with the Linear -MCP attached. The gateway resolves the user from the LiteLLM key, lists Linear's -tools with the stored per-user token, lets the model call one, executes it -upstream with that token, and returns the answer. This is proven against the -real Linear MCP server (mcp.linear.app) and a real Anthropic model, once per -documented ingress header (x-litellm-api-key and Authorization). - -The authorize dance is seeded through the mcp SDK's OAuthClientProvider; the one -step Linear cannot auto-approve is the human consent, so it is captured once out -of band (mcp/linear_session_capture.py) into a saved browser session and a -headless Chromium clicks Approve every run. The test therefore skips unless -E2E_LINEAR_STORAGE_STATE points at that session, so it never runs on the per-PR -CI path; it is a nightly/on-demand real-server smoke test. - -Fail-before-fix: without the stored per-user token the gateway lists no Linear -tools, so mcp_list_tools comes back empty, nothing is called, and the -assertions fail; a served, called, non-empty Linear tool proves the gateway -pulled and used the user's token. -""" - -from __future__ import annotations - -import os - -import pytest - -from e2e_config import CHEAP_ANTHROPIC_MODEL, LINEAR_MCP_URL, LINEAR_STORAGE_STATE, unique_marker -from e2e_http import AuthHeaders -from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, KeyGenerateBody, McpChatTool, McpServerCreateBody, ObjectPermission -from proxy_client import ProxyClient - -pytest.importorskip("mcp", reason="mcp SDK not installed; run `uv sync --inexact --group e2e-dev`") -pytest.importorskip( - "playwright.async_api", - reason="playwright not installed; run `uv pip install playwright` and `playwright install chromium`", -) - -from oauth_chat_client import ChatMcpClient, build_chat_client # noqa: E402 # imports follow the importorskip guards - -pytestmark = [ - pytest.mark.e2e, - pytest.mark.skipif( - not LINEAR_STORAGE_STATE or not os.path.exists(LINEAR_STORAGE_STATE), - reason="set E2E_LINEAR_STORAGE_STATE to a Linear session captured via mcp/linear_session_capture.py", - ), -] - -# Pinned from a live dance during verification (never guessed); the gateway -# prefixes every upstream tool name with the server alias. list_teams is a -# read-only Linear tool that takes no arguments and returns the caller's teams. -LINEAR_READONLY_TOOL = "list_teams" -LINEAR_PROMPT = "Use the list_teams tool to list my Linear teams, then reply with the name of one of them." - - -@pytest.fixture(scope="session") -def chat_client(proxy: ProxyClient) -> ChatMcpClient: - return build_chat_client(proxy) - - -class TestMcpChatCompletionOauth: - """A scoped internal-user key on a real Linear authorization_code server, - used through /chat/completions once per ingress header: the gateway pulls - the user's stored upstream token, lists and executes Linear's tools during - the completion, and returns the answer.""" - - @pytest.mark.covers( - "mcp.list_tools.oauth.succeeds", - "mcp.call_tool.oauth.succeeds", - "mcp.chat_completion.oauth.auto_executes_tools", - ) - def test_chat_completion_uses_linear_with_x_litellm_api_key_header( - self, chat_client: ChatMcpClient, resources: ResourceManager - ) -> None: - marker = unique_marker() - alias = f"e2elinear{marker}" - created = chat_client.create_server( - McpServerCreateBody( - alias=alias, - url=LINEAR_MCP_URL, - allow_all_keys=False, - auth_type="oauth2", - oauth2_flow="authorization_code", - ) - ) - resources.defer(lambda: chat_client.delete_server(created.server_id)) - - stored = chat_client.server_info(created.server_id) - assert stored.auth_type == "oauth2" - assert stored.oauth2_flow == "authorization_code" - assert stored.allow_all_keys is False - - key = chat_client.proxy.generate_key( - KeyGenerateBody( - user_id="e2e-test-user", - object_permission=ObjectPermission(mcp_servers=[created.server_id]), - ) - ) - resources.defer(lambda: chat_client.proxy.delete_key(key)) - - seeded = chat_client.seed_user_token(alias, key, LINEAR_STORAGE_STATE) - assert f"{alias}-{LINEAR_READONLY_TOOL}" in seeded, ( - f"the authorize dance listed {seeded}, expected it to include {alias}-{LINEAR_READONLY_TOOL}" - ) - - response = chat_client.chat_with_mcp( - AuthHeaders.model_validate({"x-litellm-api-key": f"Bearer {key}"}), - ChatBody( - model=CHEAP_ANTHROPIC_MODEL, - messages=[ChatMessage(role="user", content=LINEAR_PROMPT)], - tools=[ - McpChatTool( - server_url=f"litellm_proxy/mcp/{alias}", - server_label=alias, - require_approval="never", - ) - ], - ), - ) - - message = response.choices[0].message - assert message is not None and message.content, f"completion returned no answer: {response}" - meta = message.provider_specific_fields - assert meta is not None, f"no MCP metadata on the completion: {response}" - listed = {t.function.name for t in (meta.mcp_list_tools or []) if t.function} - assert f"{alias}-{LINEAR_READONLY_TOOL}" in listed, ( - f"the gateway listed {sorted(listed)}, expected the stored token to surface {alias}-{LINEAR_READONLY_TOOL}" - ) - results = [r for r in (meta.mcp_call_results or []) if r.name == f"{alias}-{LINEAR_READONLY_TOOL}"] - assert results and results[0].result, ( - f"Linear tool {alias}-{LINEAR_READONLY_TOOL} was not executed with a result: {meta.mcp_call_results}" - ) - - @pytest.mark.covers( - "mcp.list_tools.oauth.succeeds", - "mcp.call_tool.oauth.succeeds", - "mcp.chat_completion.oauth.auto_executes_tools", - ) - def test_chat_completion_uses_linear_with_authorization_bearer_header( - self, chat_client: ChatMcpClient, resources: ResourceManager - ) -> None: - marker = unique_marker() - alias = f"e2elinear{marker}" - created = chat_client.create_server( - McpServerCreateBody( - alias=alias, - url=LINEAR_MCP_URL, - allow_all_keys=False, - auth_type="oauth2", - oauth2_flow="authorization_code", - ) - ) - resources.defer(lambda: chat_client.delete_server(created.server_id)) - - stored = chat_client.server_info(created.server_id) - assert stored.auth_type == "oauth2" - assert stored.oauth2_flow == "authorization_code" - assert stored.allow_all_keys is False - - key = chat_client.proxy.generate_key( - KeyGenerateBody( - user_id="e2e-test-user", - object_permission=ObjectPermission(mcp_servers=[created.server_id]), - ) - ) - resources.defer(lambda: chat_client.proxy.delete_key(key)) - - seeded = chat_client.seed_user_token(alias, key, LINEAR_STORAGE_STATE) - assert f"{alias}-{LINEAR_READONLY_TOOL}" in seeded, ( - f"the authorize dance listed {seeded}, expected it to include {alias}-{LINEAR_READONLY_TOOL}" - ) - - response = chat_client.chat_with_mcp( - AuthHeaders.model_validate({"authorization": f"Bearer {key}"}), - ChatBody( - model=CHEAP_ANTHROPIC_MODEL, - messages=[ChatMessage(role="user", content=LINEAR_PROMPT)], - tools=[ - McpChatTool( - server_url=f"litellm_proxy/mcp/{alias}", - server_label=alias, - require_approval="never", - ) - ], - ), - ) - - message = response.choices[0].message - assert message is not None and message.content, f"completion returned no answer: {response}" - meta = message.provider_specific_fields - assert meta is not None, f"no MCP metadata on the completion: {response}" - listed = {t.function.name for t in (meta.mcp_list_tools or []) if t.function} - assert f"{alias}-{LINEAR_READONLY_TOOL}" in listed, ( - f"the gateway listed {sorted(listed)}, expected the stored token to surface {alias}-{LINEAR_READONLY_TOOL}" - ) - results = [r for r in (meta.mcp_call_results or []) if r.name == f"{alias}-{LINEAR_READONLY_TOOL}"] - assert results and results[0].result, ( - f"Linear tool {alias}-{LINEAR_READONLY_TOOL} was not executed with a result: {meta.mcp_call_results}" - )