From 2927e88e794dcbd5fff081c9d0762bac7e33a9c3 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 5 Aug 2026 16:15:24 -0700 Subject: [PATCH] fix(e2e/mcp): avoid shipping shared logs and in-repo browser sessions Bridge prompts search a unique no-hit marker so real Datadog service logs are not folded into an external model turn. Session capture defaults to temp and refuses paths under the git tree; gitignore covers leftover names --- .gitignore | 2 ++ tests/e2e/mcp/dd_session_capture.py | 26 ++++++++++++++++--- .../e2e/mcp/test_mcp_chat_completions_e2e.py | 5 ++-- tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py | 4 +-- tests/e2e/mcp/test_mcp_messages_e2e.py | 23 +++++++++------- tests/e2e/mcp/test_mcp_responses_e2e.py | 5 ++-- 6 files changed, 47 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 13f2202305d..3ebcfe51b0a 100644 --- a/.gitignore +++ b/.gitignore @@ -106,6 +106,8 @@ STABILIZATION_TODO.md **/test-results **/playwright-report **/*.storageState.json +**/.dd_session.json +**/litellm-e2e-dd-session.json **/coverage test-config diff --git a/tests/e2e/mcp/dd_session_capture.py b/tests/e2e/mcp/dd_session_capture.py index b2ed612b010..f60ce7f1bcd 100644 --- a/tests/e2e/mcp/dd_session_capture.py +++ b/tests/e2e/mcp/dd_session_capture.py @@ -1,24 +1,44 @@ """Capture a Datadog browser session for the MCP OAuth e2e tests. -Run this once to log into Datadog and save the browser session: +Run this once to log into Datadog and save the browser session outside the +repo (default: $TMPDIR/litellm-e2e-dd-session.json): uv run python tests/e2e/mcp/dd_session_capture.py Then set the env var and run the OAuth tests: - export E2E_DD_STORAGE_STATE=tests/e2e/mcp/.dd_session.json + export E2E_DD_STORAGE_STATE="$TMPDIR/litellm-e2e-dd-session.json" uv run pytest tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py -v """ from __future__ import annotations import os +import tempfile from pathlib import Path -DEFAULT_STATE_PATH = Path(__file__).parent / ".dd_session.json" +DEFAULT_STATE_PATH = Path(tempfile.gettempdir()) / "litellm-e2e-dd-session.json" + + +def _repo_root() -> Path | None: + here = Path(__file__).resolve() + for parent in here.parents: + if (parent / ".git").exists(): + return parent + return None def capture(state_path: Path) -> None: + state_path = state_path.expanduser().resolve() + repo = _repo_root() + if repo is not None and (state_path == repo or repo in state_path.parents): + raise SystemExit( + f"Refusing to write session state under the repo ({state_path}). " + f"Set E2E_DD_STORAGE_STATE to a path outside the tree " + f"(default: {DEFAULT_STATE_PATH})." + ) + state_path.parent.mkdir(parents=True, exist_ok=True) + from playwright.sync_api import sync_playwright with sync_playwright() as p: diff --git a/tests/e2e/mcp/test_mcp_chat_completions_e2e.py b/tests/e2e/mcp/test_mcp_chat_completions_e2e.py index 9f26e257318..946eb4c0ad3 100644 --- a/tests/e2e/mcp/test_mcp_chat_completions_e2e.py +++ b/tests/e2e/mcp/test_mcp_chat_completions_e2e.py @@ -38,6 +38,7 @@ class TestChatCompletionMcpAutoExecute: dd = register_datadog_mcp(client, resources) client.await_registered(dd.server_id) + marker = f"e2e-mcp-chat-nohit-{unique_marker()}" key = client.generate_key( user_id=f"e2e-mcp-chat-{unique_marker()}", mcp_servers=[dd.server_id], @@ -52,8 +53,8 @@ class TestChatCompletionMcpAutoExecute: role="user", content=( "Use the search_datadog_logs tool to search for logs " - "with query 'service:litellm' from now-30m to now with " - "max_tokens 500. After you get results, summarize what you found." + f"with query '{marker}' from now-30m to now with " + "max_tokens 100. After you get results, reply with ok only." ), ) ], diff --git a/tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py b/tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py index 0401c12d361..57d2f17cd9a 100644 --- a/tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py +++ b/tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py @@ -126,8 +126,8 @@ class TestDatadogMcpOAuth: role="user", content=( "Use the search_datadog_logs tool to search for logs " - "with query 'service:litellm' from now-30m to now with " - "max_tokens 500. After you get results, summarize what you found." + f"with query 'e2e-mcp-oauth-nohit-{marker}' from now-30m to now with " + "max_tokens 100. After you get results, reply with ok only." ), ) ], diff --git a/tests/e2e/mcp/test_mcp_messages_e2e.py b/tests/e2e/mcp/test_mcp_messages_e2e.py index da91505b0f9..0426c740068 100644 --- a/tests/e2e/mcp/test_mcp_messages_e2e.py +++ b/tests/e2e/mcp/test_mcp_messages_e2e.py @@ -30,18 +30,19 @@ from models import AnthropicMcpTool, AnthropicMessagesBody, ChatMessage pytestmark = pytest.mark.e2e -TOOL_PROMPT = ( - "Use the search_datadog_logs tool to search for logs " - "with query 'service:litellm' from now-30m to now with " - "max_tokens 500. After you get results, summarize what you found in one sentence." -) +def _tool_prompt(marker: str) -> str: + return ( + "Use the search_datadog_logs tool to search for logs " + f"with query '{marker}' from now-30m to now with " + "max_tokens 100. After you get results, reply with ok only." + ) -def _messages_body(model: str, alias: str) -> AnthropicMessagesBody: +def _messages_body(model: str, alias: str, marker: str) -> AnthropicMessagesBody: return AnthropicMessagesBody( model=model, max_tokens=1024, - messages=[ChatMessage(role="user", content=TOOL_PROMPT)], + messages=[ChatMessage(role="user", content=_tool_prompt(marker))], tools=[ AnthropicMcpTool( server_label="datadog", @@ -63,6 +64,7 @@ class TestMessagesMcpAutoExecute: dd = register_datadog_mcp(client, resources) client.await_registered(dd.server_id) + marker = f"e2e-mcp-msg-nohit-{unique_marker()}" key = client.generate_key( user_id=f"e2e-mcp-msg-{unique_marker()}", mcp_servers=[dd.server_id], @@ -71,7 +73,9 @@ class TestMessagesMcpAutoExecute: resources.defer(lambda: client.proxy.delete_key(key)) response = unwrap( - client.messages_with_mcp(key, _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias)) + client.messages_with_mcp( + key, _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias, marker) + ) ) assert response.content, f"/v1/messages returned no content blocks: {response}" @@ -91,6 +95,7 @@ class TestMessagesMcpAutoExecute: dd = register_datadog_mcp(client, resources) client.await_registered(dd.server_id) + marker = f"e2e-mcp-msg-stream-nohit-{unique_marker()}" key = client.generate_key( user_id=f"e2e-mcp-msg-stream-{unique_marker()}", mcp_servers=[dd.server_id], @@ -98,7 +103,7 @@ class TestMessagesMcpAutoExecute: ) resources.defer(lambda: client.proxy.delete_key(key)) - body = _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias) + body = _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias, marker) body.stream = True result = client.messages_stream_with_mcp(key, body) diff --git a/tests/e2e/mcp/test_mcp_responses_e2e.py b/tests/e2e/mcp/test_mcp_responses_e2e.py index 7991dd4d080..234bfb7ec11 100644 --- a/tests/e2e/mcp/test_mcp_responses_e2e.py +++ b/tests/e2e/mcp/test_mcp_responses_e2e.py @@ -35,6 +35,7 @@ class TestResponsesMcpAutoExecute: dd = register_datadog_mcp(client, resources) client.await_registered(dd.server_id) + marker = f"e2e-mcp-resp-nohit-{unique_marker()}" key = client.generate_key( user_id=f"e2e-mcp-resp-{unique_marker()}", mcp_servers=[dd.server_id], @@ -48,8 +49,8 @@ class TestResponsesMcpAutoExecute: ResponsesMcpInputMessage( content=( "Use the search_datadog_logs tool to search for logs " - "with query 'service:litellm' from now-30m to now with " - "max_tokens 500. After you get results, summarize what you found." + f"with query '{marker}' from now-30m to now with " + "max_tokens 100. After you get results, reply with ok only." ) ) ],