fix(e2e/mcp): avoid shipping shared logs and in-repo browser sessions

Bridge prompts search a unique no-hit marker so real Datadog service logs
are not folded into an external model turn. Session capture defaults to
temp and refuses paths under the git tree; gitignore covers leftover names
This commit is contained in:
mubashir1osmani 2026-08-05 16:15:24 -07:00
parent 3e8774a3fc
commit 2927e88e79
6 changed files with 47 additions and 18 deletions

2
.gitignore vendored
View file

@ -106,6 +106,8 @@ STABILIZATION_TODO.md
**/test-results
**/playwright-report
**/*.storageState.json
**/.dd_session.json
**/litellm-e2e-dd-session.json
**/coverage
test-config

View file

@ -1,24 +1,44 @@
"""Capture a Datadog browser session for the MCP OAuth e2e tests.
Run this once to log into Datadog and save the browser session:
Run this once to log into Datadog and save the browser session outside the
repo (default: $TMPDIR/litellm-e2e-dd-session.json):
uv run python tests/e2e/mcp/dd_session_capture.py
Then set the env var and run the OAuth tests:
export E2E_DD_STORAGE_STATE=tests/e2e/mcp/.dd_session.json
export E2E_DD_STORAGE_STATE="$TMPDIR/litellm-e2e-dd-session.json"
uv run pytest tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py -v
"""
from __future__ import annotations
import os
import tempfile
from pathlib import Path
DEFAULT_STATE_PATH = Path(__file__).parent / ".dd_session.json"
DEFAULT_STATE_PATH = Path(tempfile.gettempdir()) / "litellm-e2e-dd-session.json"
def _repo_root() -> Path | None:
here = Path(__file__).resolve()
for parent in here.parents:
if (parent / ".git").exists():
return parent
return None
def capture(state_path: Path) -> None:
state_path = state_path.expanduser().resolve()
repo = _repo_root()
if repo is not None and (state_path == repo or repo in state_path.parents):
raise SystemExit(
f"Refusing to write session state under the repo ({state_path}). "
f"Set E2E_DD_STORAGE_STATE to a path outside the tree "
f"(default: {DEFAULT_STATE_PATH})."
)
state_path.parent.mkdir(parents=True, exist_ok=True)
from playwright.sync_api import sync_playwright
with sync_playwright() as p:

View file

@ -38,6 +38,7 @@ class TestChatCompletionMcpAutoExecute:
dd = register_datadog_mcp(client, resources)
client.await_registered(dd.server_id)
marker = f"e2e-mcp-chat-nohit-{unique_marker()}"
key = client.generate_key(
user_id=f"e2e-mcp-chat-{unique_marker()}",
mcp_servers=[dd.server_id],
@ -52,8 +53,8 @@ class TestChatCompletionMcpAutoExecute:
role="user",
content=(
"Use the search_datadog_logs tool to search for logs "
"with query 'service:litellm' from now-30m to now with "
"max_tokens 500. After you get results, summarize what you found."
f"with query '{marker}' from now-30m to now with "
"max_tokens 100. After you get results, reply with ok only."
),
)
],

View file

@ -126,8 +126,8 @@ class TestDatadogMcpOAuth:
role="user",
content=(
"Use the search_datadog_logs tool to search for logs "
"with query 'service:litellm' from now-30m to now with "
"max_tokens 500. After you get results, summarize what you found."
f"with query 'e2e-mcp-oauth-nohit-{marker}' from now-30m to now with "
"max_tokens 100. After you get results, reply with ok only."
),
)
],

View file

@ -30,18 +30,19 @@ from models import AnthropicMcpTool, AnthropicMessagesBody, ChatMessage
pytestmark = pytest.mark.e2e
TOOL_PROMPT = (
"Use the search_datadog_logs tool to search for logs "
"with query 'service:litellm' from now-30m to now with "
"max_tokens 500. After you get results, summarize what you found in one sentence."
)
def _tool_prompt(marker: str) -> str:
return (
"Use the search_datadog_logs tool to search for logs "
f"with query '{marker}' from now-30m to now with "
"max_tokens 100. After you get results, reply with ok only."
)
def _messages_body(model: str, alias: str) -> AnthropicMessagesBody:
def _messages_body(model: str, alias: str, marker: str) -> AnthropicMessagesBody:
return AnthropicMessagesBody(
model=model,
max_tokens=1024,
messages=[ChatMessage(role="user", content=TOOL_PROMPT)],
messages=[ChatMessage(role="user", content=_tool_prompt(marker))],
tools=[
AnthropicMcpTool(
server_label="datadog",
@ -63,6 +64,7 @@ class TestMessagesMcpAutoExecute:
dd = register_datadog_mcp(client, resources)
client.await_registered(dd.server_id)
marker = f"e2e-mcp-msg-nohit-{unique_marker()}"
key = client.generate_key(
user_id=f"e2e-mcp-msg-{unique_marker()}",
mcp_servers=[dd.server_id],
@ -71,7 +73,9 @@ class TestMessagesMcpAutoExecute:
resources.defer(lambda: client.proxy.delete_key(key))
response = unwrap(
client.messages_with_mcp(key, _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias))
client.messages_with_mcp(
key, _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias, marker)
)
)
assert response.content, f"/v1/messages returned no content blocks: {response}"
@ -91,6 +95,7 @@ class TestMessagesMcpAutoExecute:
dd = register_datadog_mcp(client, resources)
client.await_registered(dd.server_id)
marker = f"e2e-mcp-msg-stream-nohit-{unique_marker()}"
key = client.generate_key(
user_id=f"e2e-mcp-msg-stream-{unique_marker()}",
mcp_servers=[dd.server_id],
@ -98,7 +103,7 @@ class TestMessagesMcpAutoExecute:
)
resources.defer(lambda: client.proxy.delete_key(key))
body = _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias)
body = _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias, marker)
body.stream = True
result = client.messages_stream_with_mcp(key, body)

View file

@ -35,6 +35,7 @@ class TestResponsesMcpAutoExecute:
dd = register_datadog_mcp(client, resources)
client.await_registered(dd.server_id)
marker = f"e2e-mcp-resp-nohit-{unique_marker()}"
key = client.generate_key(
user_id=f"e2e-mcp-resp-{unique_marker()}",
mcp_servers=[dd.server_id],
@ -48,8 +49,8 @@ class TestResponsesMcpAutoExecute:
ResponsesMcpInputMessage(
content=(
"Use the search_datadog_logs tool to search for logs "
"with query 'service:litellm' from now-30m to now with "
"max_tokens 500. After you get results, summarize what you found."
f"with query '{marker}' from now-30m to now with "
"max_tokens 100. After you get results, reply with ok only."
)
)
],