mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(e2e/mcp): avoid shipping shared logs and in-repo browser sessions
Bridge prompts search a unique no-hit marker so real Datadog service logs are not folded into an external model turn. Session capture defaults to temp and refuses paths under the git tree; gitignore covers leftover names
This commit is contained in:
parent
3e8774a3fc
commit
2927e88e79
6 changed files with 47 additions and 18 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -106,6 +106,8 @@ STABILIZATION_TODO.md
|
|||
**/test-results
|
||||
**/playwright-report
|
||||
**/*.storageState.json
|
||||
**/.dd_session.json
|
||||
**/litellm-e2e-dd-session.json
|
||||
**/coverage
|
||||
test-config
|
||||
|
||||
|
|
|
|||
|
|
@ -1,24 +1,44 @@
|
|||
"""Capture a Datadog browser session for the MCP OAuth e2e tests.
|
||||
|
||||
Run this once to log into Datadog and save the browser session:
|
||||
Run this once to log into Datadog and save the browser session outside the
|
||||
repo (default: $TMPDIR/litellm-e2e-dd-session.json):
|
||||
|
||||
uv run python tests/e2e/mcp/dd_session_capture.py
|
||||
|
||||
Then set the env var and run the OAuth tests:
|
||||
|
||||
export E2E_DD_STORAGE_STATE=tests/e2e/mcp/.dd_session.json
|
||||
export E2E_DD_STORAGE_STATE="$TMPDIR/litellm-e2e-dd-session.json"
|
||||
uv run pytest tests/e2e/mcp/test_mcp_datadog_oauth_e2e.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_STATE_PATH = Path(__file__).parent / ".dd_session.json"
|
||||
DEFAULT_STATE_PATH = Path(tempfile.gettempdir()) / "litellm-e2e-dd-session.json"
|
||||
|
||||
|
||||
def _repo_root() -> Path | None:
|
||||
here = Path(__file__).resolve()
|
||||
for parent in here.parents:
|
||||
if (parent / ".git").exists():
|
||||
return parent
|
||||
return None
|
||||
|
||||
|
||||
def capture(state_path: Path) -> None:
|
||||
state_path = state_path.expanduser().resolve()
|
||||
repo = _repo_root()
|
||||
if repo is not None and (state_path == repo or repo in state_path.parents):
|
||||
raise SystemExit(
|
||||
f"Refusing to write session state under the repo ({state_path}). "
|
||||
f"Set E2E_DD_STORAGE_STATE to a path outside the tree "
|
||||
f"(default: {DEFAULT_STATE_PATH})."
|
||||
)
|
||||
state_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ class TestChatCompletionMcpAutoExecute:
|
|||
dd = register_datadog_mcp(client, resources)
|
||||
client.await_registered(dd.server_id)
|
||||
|
||||
marker = f"e2e-mcp-chat-nohit-{unique_marker()}"
|
||||
key = client.generate_key(
|
||||
user_id=f"e2e-mcp-chat-{unique_marker()}",
|
||||
mcp_servers=[dd.server_id],
|
||||
|
|
@ -52,8 +53,8 @@ class TestChatCompletionMcpAutoExecute:
|
|||
role="user",
|
||||
content=(
|
||||
"Use the search_datadog_logs tool to search for logs "
|
||||
"with query 'service:litellm' from now-30m to now with "
|
||||
"max_tokens 500. After you get results, summarize what you found."
|
||||
f"with query '{marker}' from now-30m to now with "
|
||||
"max_tokens 100. After you get results, reply with ok only."
|
||||
),
|
||||
)
|
||||
],
|
||||
|
|
|
|||
|
|
@ -126,8 +126,8 @@ class TestDatadogMcpOAuth:
|
|||
role="user",
|
||||
content=(
|
||||
"Use the search_datadog_logs tool to search for logs "
|
||||
"with query 'service:litellm' from now-30m to now with "
|
||||
"max_tokens 500. After you get results, summarize what you found."
|
||||
f"with query 'e2e-mcp-oauth-nohit-{marker}' from now-30m to now with "
|
||||
"max_tokens 100. After you get results, reply with ok only."
|
||||
),
|
||||
)
|
||||
],
|
||||
|
|
|
|||
|
|
@ -30,18 +30,19 @@ from models import AnthropicMcpTool, AnthropicMessagesBody, ChatMessage
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
TOOL_PROMPT = (
|
||||
"Use the search_datadog_logs tool to search for logs "
|
||||
"with query 'service:litellm' from now-30m to now with "
|
||||
"max_tokens 500. After you get results, summarize what you found in one sentence."
|
||||
)
|
||||
def _tool_prompt(marker: str) -> str:
|
||||
return (
|
||||
"Use the search_datadog_logs tool to search for logs "
|
||||
f"with query '{marker}' from now-30m to now with "
|
||||
"max_tokens 100. After you get results, reply with ok only."
|
||||
)
|
||||
|
||||
|
||||
def _messages_body(model: str, alias: str) -> AnthropicMessagesBody:
|
||||
def _messages_body(model: str, alias: str, marker: str) -> AnthropicMessagesBody:
|
||||
return AnthropicMessagesBody(
|
||||
model=model,
|
||||
max_tokens=1024,
|
||||
messages=[ChatMessage(role="user", content=TOOL_PROMPT)],
|
||||
messages=[ChatMessage(role="user", content=_tool_prompt(marker))],
|
||||
tools=[
|
||||
AnthropicMcpTool(
|
||||
server_label="datadog",
|
||||
|
|
@ -63,6 +64,7 @@ class TestMessagesMcpAutoExecute:
|
|||
dd = register_datadog_mcp(client, resources)
|
||||
client.await_registered(dd.server_id)
|
||||
|
||||
marker = f"e2e-mcp-msg-nohit-{unique_marker()}"
|
||||
key = client.generate_key(
|
||||
user_id=f"e2e-mcp-msg-{unique_marker()}",
|
||||
mcp_servers=[dd.server_id],
|
||||
|
|
@ -71,7 +73,9 @@ class TestMessagesMcpAutoExecute:
|
|||
resources.defer(lambda: client.proxy.delete_key(key))
|
||||
|
||||
response = unwrap(
|
||||
client.messages_with_mcp(key, _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias))
|
||||
client.messages_with_mcp(
|
||||
key, _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias, marker)
|
||||
)
|
||||
)
|
||||
|
||||
assert response.content, f"/v1/messages returned no content blocks: {response}"
|
||||
|
|
@ -91,6 +95,7 @@ class TestMessagesMcpAutoExecute:
|
|||
dd = register_datadog_mcp(client, resources)
|
||||
client.await_registered(dd.server_id)
|
||||
|
||||
marker = f"e2e-mcp-msg-stream-nohit-{unique_marker()}"
|
||||
key = client.generate_key(
|
||||
user_id=f"e2e-mcp-msg-stream-{unique_marker()}",
|
||||
mcp_servers=[dd.server_id],
|
||||
|
|
@ -98,7 +103,7 @@ class TestMessagesMcpAutoExecute:
|
|||
)
|
||||
resources.defer(lambda: client.proxy.delete_key(key))
|
||||
|
||||
body = _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias)
|
||||
body = _messages_body(CHEAP_ANTHROPIC_MODEL, dd.alias, marker)
|
||||
body.stream = True
|
||||
|
||||
result = client.messages_stream_with_mcp(key, body)
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ class TestResponsesMcpAutoExecute:
|
|||
dd = register_datadog_mcp(client, resources)
|
||||
client.await_registered(dd.server_id)
|
||||
|
||||
marker = f"e2e-mcp-resp-nohit-{unique_marker()}"
|
||||
key = client.generate_key(
|
||||
user_id=f"e2e-mcp-resp-{unique_marker()}",
|
||||
mcp_servers=[dd.server_id],
|
||||
|
|
@ -48,8 +49,8 @@ class TestResponsesMcpAutoExecute:
|
|||
ResponsesMcpInputMessage(
|
||||
content=(
|
||||
"Use the search_datadog_logs tool to search for logs "
|
||||
"with query 'service:litellm' from now-30m to now with "
|
||||
"max_tokens 500. After you get results, summarize what you found."
|
||||
f"with query '{marker}' from now-30m to now with "
|
||||
"max_tokens 100. After you get results, reply with ok only."
|
||||
)
|
||||
)
|
||||
],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue