mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-05 02:41:56 +00:00
fdf380d0e3
* test(e2e): harden stage flakes for batches, UI, and MCP Unique batch model names avoid load-balancing onto stale azure-batch deployments that still pointed at the retired gpt-4.1-mini-batch, which only the managed/unified path was hitting. Retry batch retrieve on 500 and /ui/api-keys navigation on ERR_ABORTED. Skip the MCP key-access suite when the compose-only mcp-upstream is unreachable on stage k8s * test(e2e): cover Datadog remote MCP via search_datadog_logs Register the regional Datadog MCP endpoint with DD-API-KEY / DD-APPLICATION-KEY static headers (CI-safe header auth; browser OAuth is not headless-automatable). Seed a chat completion marked e2e-datadog-mcp-*, assert the proxy shipped it, list tools, call search_datadog_logs for the marker, and delete the server on teardown. Math-upstream key-access tests only skip when that compose service is unreachable * test(e2e): drop compose math MCP upstream; use Datadog only Key-access denial and happy-path MCP e2e both register the real regional Datadog remote MCP server with DD-API-KEY / DD-APPLICATION-KEY headers. Remove the mcp-upstream compose service and FastMCP add/multiply fixture * docs(e2e): require real Datadog MCP for all mcp suite tests Document that tests/e2e/mcp must register via datadog_mcp helpers against mcp.<site>/v1/mcp and must not introduce compose or fake MCP upstreams * chore: restore mcp_e2e_upstream_server.py Keep the FastMCP fixture file; e2e no longer wires it in compose, but the module itself is not part of the Datadog-only cleanup * fix(e2e): load tests/e2e/.env and fix datadog_reader importlib load pytest on the host never inherited compose env_file keys, so DD_API_KEY stayed empty. load_dotenv tests/e2e/.env in e2e_config. Register the dynamically loaded datadog_reader module in sys.modules so dataclasses do not crash under Python 3.12 * test(e2e/batches): harden azure/vertex unified lifecycle flakes Put the provider deployment name in every JSONL body so Azure does not depend on a perfect model rewrite. Retry create/retrieve/cancel on transient statuses with backoff. Drop cancel assertions for azure and vertex (registry only has a shared basic cell; create+retrieve prove routing, cancel stays best-effort cleanup) * test(e2e/ui): treat api-keys shell as success after SPA ERR_ABORTED Post-login client redirects abort the first /ui/api-keys/ goto on stage. Wait off /ui/login after cookie set, then accept the page once Create New Key is visible even if goto raised ERR_ABORTED * test(e2e): drop flaky key models dropdown Playwright suite API management e2e already covers key generate/update persistence. The UI Models-dropdown sentinel cases only added SPA ERR_ABORTED noise and no unique product signal. Remove the suite and unused browser fixtures
48 lines
1.3 KiB
Python
48 lines
1.3 KiB
Python
"""Shared helpers for e2e tests that register the real Datadog remote MCP server."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from e2e_config import datadog_mcp_url, unique_marker
|
|
from lifecycle import ResourceManager
|
|
from mcp_client import McpClient
|
|
|
|
SEARCH_LOGS_TOOL = "search_datadog_logs"
|
|
|
|
|
|
def _dd_api_key() -> str:
|
|
return os.environ.get("DD_API_KEY", "").strip()
|
|
|
|
|
|
def _dd_app_key() -> str:
|
|
return os.environ.get("DD_APP_KEY", "").strip()
|
|
|
|
|
|
def assert_dd_mcp_creds() -> None:
|
|
if not _dd_api_key() or not _dd_app_key():
|
|
import pytest
|
|
|
|
pytest.fail(
|
|
"Datadog MCP e2e requires DD_API_KEY and DD_APP_KEY "
|
|
"(header auth to mcp.<site>/v1/mcp; on the cluster the secret manager "
|
|
"injects them, locally tests/e2e/.env)"
|
|
)
|
|
|
|
|
|
def register_datadog_mcp(client: McpClient, resources: ResourceManager) -> str:
|
|
assert_dd_mcp_creds()
|
|
name = f"e2e_dd_mcp_{unique_marker()}"
|
|
server_id = client.register_server(
|
|
server_name=name,
|
|
alias=name,
|
|
url=datadog_mcp_url(toolsets="core"),
|
|
transport="http",
|
|
static_headers={
|
|
"DD-API-KEY": _dd_api_key(),
|
|
"DD-APPLICATION-KEY": _dd_app_key(),
|
|
},
|
|
allowed_tools=[SEARCH_LOGS_TOOL],
|
|
)
|
|
resources.defer(lambda: client.delete_server(server_id))
|
|
return server_id
|