diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 740540b25bc..0c7cb39aef1 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -28,9 +28,7 @@ MASTER_KEY = os.environ.get("LITELLM_MASTER_KEY", "sk-1234") # single path-routing host (stage ALB, compose monolith) works for both planes. # Set LITELLM_CONTROL_PLANE_URL only when management is a different base than # the LLM host and you are not going through an ingress that path-routes. -CONTROL_PLANE_BASE_URL = os.environ.get( - "LITELLM_CONTROL_PLANE_URL", PROXY_BASE_URL -).rstrip("/") +CONTROL_PLANE_BASE_URL = os.environ.get("LITELLM_CONTROL_PLANE_URL", PROXY_BASE_URL).rstrip("/") def parse_replica_urls(raw: str, fallback: str) -> tuple[str, ...]: @@ -52,6 +50,7 @@ CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp") LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "") +LINEAR_READONLY_TOOL: Final = "list_teams" # Jaeger query API of the compose stack's OTEL trace destination (the `jaeger` # service in docker-compose.yml maps it to host 16686). Trace-completeness tests @@ -106,18 +105,13 @@ PROPAGATION_TIMEOUT = float(os.environ.get("E2E_PROPAGATION_TIMEOUT", "15")) # for empty values) means the harness behaves exactly as before this knob # existed. FIXTURE_MODE_RAW = os.environ.get("E2E_FIXTURE_MODE", "live") -FIXTURE_DIR = Path( - os.environ.get("E2E_FIXTURE_DIR", "").strip() - or str(Path(__file__).resolve().parent / ".fixtures") -) +FIXTURE_DIR = Path(os.environ.get("E2E_FIXTURE_DIR", "").strip() or str(Path(__file__).resolve().parent / ".fixtures")) # Where the provider-edge server binds, and the host name edge api_base URLs # advertise to the proxy. They differ when the proxy runs in a container and # reaches the pytest host via a gateway name like host.docker.internal. PROVIDER_EDGE_BIND_HOST = os.environ.get("E2E_PROVIDER_EDGE_BIND_HOST", "").strip() or "127.0.0.1" -PROVIDER_EDGE_ADVERTISE_HOST = ( - os.environ.get("E2E_PROVIDER_EDGE_ADVERTISE_HOST", "").strip() or PROVIDER_EDGE_BIND_HOST -) +PROVIDER_EDGE_ADVERTISE_HOST = os.environ.get("E2E_PROVIDER_EDGE_ADVERTISE_HOST", "").strip() or PROVIDER_EDGE_BIND_HOST # Deliberately modest concurrency. The suite shares its proxy with every other # suite in the run, and 750 users at spawn rate 50 saturated the request path hard @@ -149,18 +143,10 @@ ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) ANOMALY_MAX_ERROR_RATIO = float(os.environ.get("E2E_ANOMALY_MAX_ERROR_RATIO", "0.05")) -ANOMALY_MIN_WARM_CACHE_READ_SHARE = float( - os.environ.get("E2E_ANOMALY_MIN_WARM_CACHE_READ_SHARE", "0.65") -) -ANOMALY_MAX_P95_TURN_SECONDS = float( - os.environ.get("E2E_ANOMALY_MAX_P95_TURN_SECONDS", "30") -) -ANOMALY_MAX_KEY_SPEND_USD = float( - os.environ.get("E2E_ANOMALY_MAX_KEY_SPEND_USD", "0.60") -) -ANOMALY_SPEND_SETTLE_SECONDS = float( - os.environ.get("E2E_ANOMALY_SPEND_SETTLE_SECONDS", "75") -) +ANOMALY_MIN_WARM_CACHE_READ_SHARE = float(os.environ.get("E2E_ANOMALY_MIN_WARM_CACHE_READ_SHARE", "0.65")) +ANOMALY_MAX_P95_TURN_SECONDS = float(os.environ.get("E2E_ANOMALY_MAX_P95_TURN_SECONDS", "30")) +ANOMALY_MAX_KEY_SPEND_USD = float(os.environ.get("E2E_ANOMALY_MAX_KEY_SPEND_USD", "0.60")) +ANOMALY_SPEND_SETTLE_SECONDS = float(os.environ.get("E2E_ANOMALY_SPEND_SETTLE_SECONDS", "75")) MEMORY_REQUESTS_PER_PHASE = int(os.environ.get("E2E_MEMORY_REQUESTS_PER_PHASE", "300")) MEMORY_RETRIES_PER_REQUEST = int(os.environ.get("E2E_MEMORY_RETRIES_PER_REQUEST", "2")) MEMORY_TRANSCRIPT_TURNS = int(os.environ.get("E2E_MEMORY_TRANSCRIPT_TURNS", "40")) @@ -188,8 +174,12 @@ def datadog_mcp_url(*, toolsets: str = "core") -> str: belong to a non-US1 org. """ site = ( - os.environ.get("DD_SITE", DD_SITE) or "datadoghq.com" - ).strip().removeprefix("https://").removeprefix("http://").rstrip("/") + (os.environ.get("DD_SITE", DD_SITE) or "datadoghq.com") + .strip() + .removeprefix("https://") + .removeprefix("http://") + .rstrip("/") + ) site = site.removeprefix("app.") host = "mcp.datadoghq.com" if site in ("", "datadoghq.com") else f"mcp.{site}" base = f"https://{host}/v1/mcp" diff --git a/tests/e2e/mcp/oauth_chat_client.py b/tests/e2e/mcp/oauth_chat_client.py index 1b437fea76a..ebae8029a47 100644 --- a/tests/e2e/mcp/oauth_chat_client.py +++ b/tests/e2e/mcp/oauth_chat_client.py @@ -337,6 +337,8 @@ class ChatMcpClient: base_url, ) ) + except AssertionError: + raise except Exception as exc: # noqa: BLE001 - retried to the deadline; the last error surfaces below last_error = exc time.sleep(self.proxy.poll_interval) diff --git a/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py b/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py index 01e94f7b86f..086ec929a17 100644 --- a/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py +++ b/tests/e2e/mcp/test_mcp_chat_completion_oauth_e2e.py @@ -27,8 +27,13 @@ from __future__ import annotations import os import pytest - -from e2e_config import CHEAP_ANTHROPIC_MODEL, LINEAR_MCP_URL, LINEAR_STORAGE_STATE, unique_marker +from e2e_config import ( + CHEAP_ANTHROPIC_MODEL, + LINEAR_MCP_URL, + LINEAR_READONLY_TOOL, + LINEAR_STORAGE_STATE, + unique_marker, +) from e2e_http import AuthHeaders from lifecycle import ResourceManager from models import ChatBody, ChatMessage, KeyGenerateBody, McpChatTool, McpServerCreateBody, ObjectPermission @@ -50,10 +55,6 @@ pytestmark = [ ), ] -# Pinned from a live dance during verification (never guessed); the gateway -# prefixes every upstream tool name with the server alias. list_teams is a -# read-only Linear tool that takes no arguments and returns the caller's teams. -LINEAR_READONLY_TOOL = "list_teams" LINEAR_PROMPT = "Use the list_teams tool to list my Linear teams, then reply with the name of one of them." diff --git a/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py b/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py index b35cadd7d55..1b2cc0032bb 100644 --- a/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py +++ b/tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py @@ -16,6 +16,7 @@ from typing import Final import pytest from e2e_config import ( LINEAR_MCP_URL, + LINEAR_READONLY_TOOL, LINEAR_STORAGE_STATE, PROXY_BASE_URL, PROXY_REPLICA_URLS, @@ -34,7 +35,6 @@ pytest.importorskip( from idp import Identity, Keycloak # noqa: E402 from oauth_chat_client import ChatMcpClient, InMemoryTokenStorage, build_chat_client # noqa: E402 -from test_mcp_chat_completion_oauth_e2e import LINEAR_READONLY_TOOL # noqa: E402 pytestmark = [pytest.mark.e2e, pytest.mark.mcp_oauth_live]