diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 180639b53e3..680e0dff67b 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO; marked `load` so the parent conftest collects it last and it never perturbs latency-sensitive suites. Also home of the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`): Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; additionally marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, because it spends real provider money (driven by `.github/workflows/weekly_load_anomaly.yml`) +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What remains here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`) and markerless harness unit tests for the Locust/session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/llm_translation/passthrough_client.py b/tests/e2e/llm_translation/passthrough_client.py index 3744f33e345..439594f3624 100644 --- a/tests/e2e/llm_translation/passthrough_client.py +++ b/tests/e2e/llm_translation/passthrough_client.py @@ -113,12 +113,6 @@ class OpenAIChatBody(BaseModel): max_completion_tokens: int = 64 -class VllmChatBody(BaseModel): - model: str - messages: list[ChatMessage] - max_tokens: int = 64 - - def _tags_header(tags: list[str] | None) -> str | None: return ",".join(tags) if tags else None @@ -215,19 +209,5 @@ class PassthroughClient: ), ) - def vllm_chat( - self, key: str, model: str, text: str, *, max_tokens: int = 64 - ) -> StreamingResponse: - return self.proxy.transport.send( - "/vllm/v1/chat/completions", - headers=self.proxy.transport.bearer(key), - json=VllmChatBody( - model=model, - max_tokens=max_tokens, - messages=[ChatMessage(role="user", content=text)], - ), - ) - - def build_client(proxy: ProxyClient) -> PassthroughClient: return PassthroughClient(proxy=proxy) diff --git a/tests/e2e/llm_translation/test_vllm_passthrough_e2e.py b/tests/e2e/llm_translation/test_vllm_passthrough_e2e.py deleted file mode 100644 index 3ea7d3147c1..00000000000 --- a/tests/e2e/llm_translation/test_vllm_passthrough_e2e.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Live e2e for the /vllm passthrough route. - -/vllm/{endpoint} is a raw passthrough: the client sends an OpenAI-format request -and litellm forwards it verbatim to the configured vLLM backend (VLLM_API_BASE), -with no per-request model registration (unlike the managed hosted_vllm path in -tests/e2e/batches). This drives /vllm/v1/chat/completions and asserts the -forwarded completion comes back with real content. - -On stage the backend is a CPU llama.cpp server standing in for vLLM (the cluster -is GPU-less and its CPU nodes lack the AVX512 vLLM's CPU build needs); from -litellm's side the passthrough code path is identical. Batch and file passthrough -(/vllm/v1/batches, /vllm/v1/files) is not covered: no self-hosted -vLLM-compatible server implements the OpenAI Batch API, so there is no backend to -forward those routes to. - -A passthrough call returning non-2xx fails hard (never a skip); once it is 2xx, a -missing or empty completion fails too. -""" - -import pytest - -from e2e_config import unique_marker -from e2e_http import require_successful_call -from models import ChatResponse -from passthrough_client import PassthroughClient - -pytestmark = pytest.mark.e2e - -VLLM_PASSTHROUGH_MODEL = "qwen2.5-0.5b-instruct" - - -class TestVllmChatPassthrough: - @pytest.mark.covers("llm.chat_completions.hosted_vllm.passthrough.nonstream.works") - def test_vllm_chat_passthrough_returns_completion( - self, client: PassthroughClient, scoped_key: str - ) -> None: - result = client.vllm_chat( - scoped_key, VLLM_PASSTHROUGH_MODEL, f"Say hello in one word ({unique_marker()})" - ) - require_successful_call(result) - - parsed = ChatResponse.model_validate_json(result.body) - assert parsed.choices, f"/vllm chat passthrough returned no choices: {result.body[:300]}" - message = parsed.choices[0].message - content = (message.content if message else None) or "" - assert content.strip(), f"/vllm chat passthrough returned empty content: {result.body[:300]}" diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index 89a571af83e..3a926ef2a61 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -1,24 +1,13 @@ from __future__ import annotations import os -from collections.abc import Iterator import pytest -from requests import RequestException from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV -from e2e_http import NoBody, Success from load_client import LoadClient, build_client -from load_constants import LOAD_MODEL -from models import KeyGenerateBody, LiteLLMParamsBody, ModelsListResponse -from lifecycle import ResourceManager from proxy_client import ProxyClient -LOAD_MODEL_PARAMS = LiteLLMParamsBody( - model="openai/load-mock", - mock_response="This is a mock response for the throughput load test.", -) - def pytest_collection_modifyitems( config: pytest.Config, items: list[pytest.Item] @@ -39,46 +28,3 @@ def pytest_collection_modifyitems( @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> LoadClient: return build_client(proxy) - - -def _model_is_servable(proxy: ProxyClient, model_name: str) -> bool: - result = proxy.transport.get( - "/v1/models", - headers=proxy.transport.master, - params=NoBody(), - response_type=ModelsListResponse, - ) - return isinstance(result, Success) and any(entry.id == model_name for entry in result.data.data) - - -@pytest.fixture(scope="session") -def ensure_load_model(client: LoadClient) -> Iterator[None]: - proxy = client.proxy - if _model_is_servable(proxy, LOAD_MODEL): - yield - return - - try: - model_id = proxy.create_model(LOAD_MODEL, LOAD_MODEL_PARAMS) - except (AssertionError, RequestException) as exc: - if _model_is_servable(proxy, LOAD_MODEL): - yield - return - raise AssertionError( - f"failed to register {LOAD_MODEL!r} for the throughput load test " - f"(not listed on the data plane and /model/new failed): {exc}" - ) from exc - - try: - yield - finally: - proxy.delete_model(model_id) - - -@pytest.fixture -def load_key( - resources: ResourceManager, client: LoadClient, ensure_load_model: None -) -> str: - key = client.proxy.generate_key(KeyGenerateBody(models=[LOAD_MODEL], user_id="e2e-load")) - resources.defer(lambda: client.proxy.delete_key(key)) - return key diff --git a/tests/e2e/load/load_constants.py b/tests/e2e/load/load_constants.py deleted file mode 100644 index fd97f1398f4..00000000000 --- a/tests/e2e/load/load_constants.py +++ /dev/null @@ -1,3 +0,0 @@ -from __future__ import annotations - -LOAD_MODEL = "load-mock" diff --git a/tests/e2e/load/locust_load.py b/tests/e2e/load/locust_load.py index fc1ed490186..e0da8ba70c2 100644 --- a/tests/e2e/load/locust_load.py +++ b/tests/e2e/load/locust_load.py @@ -1,18 +1,12 @@ from __future__ import annotations import csv -import os -import subprocess -import sys -import tempfile from dataclasses import dataclass from itertools import accumulate from pathlib import Path from pydantic import BaseModel, TypeAdapter -_LOCUSTFILE = Path(__file__).with_name("locustfile.py") -_CSV_PREFIX = "locust" _GENERATOR_SATURATION_MARKER = "CPU usage above" _MAX_REPORTED_ERRORS = 5 @@ -127,62 +121,3 @@ def read_generator_warnings(stderr: str) -> tuple[str, ...]: if _GENERATOR_SATURATION_MARKER in line ) return tuple(dict.fromkeys(saturated)) - - -def run_chat_load( - *, - base_url: str, - api_key: str, - model: str, - users: int, - spawn_rate: float, - duration_seconds: float, -) -> LoadResult: - with tempfile.TemporaryDirectory(prefix="e2e-load-") as report_dir: - csv_prefix = Path(report_dir) / _CSV_PREFIX - completed = subprocess.run( - [ - sys.executable, - "-m", - "locust", - "--headless", - "--json", - "--csv", - str(csv_prefix), - "--locustfile", - str(_LOCUSTFILE), - "--host", - base_url, - "--users", - str(users), - "--spawn-rate", - str(spawn_rate), - "--run-time", - f"{int(duration_seconds)}s", - "--exit-code-on-error", - "0", - ], - env={**os.environ, "LOAD_API_KEY": api_key, "LOAD_MODEL": model}, - capture_output=True, - text=True, - timeout=duration_seconds + 120, - check=False, - ) - if completed.returncode != 0: - raise RuntimeError( - f"locust exited {completed.returncode} before it could report throughput " - f"(a startup failure, not request failures, which are folded into the JSON summary via " - f"--exit-code-on-error 0):\n{completed.stderr}" - ) - try: - entries = _STATS_ADAPTER.validate_json(completed.stdout) - except ValueError as exc: - raise RuntimeError( - f"locust exited 0 but did not print a parseable --json throughput summary on stdout; " - f"got stdout={completed.stdout!r}, stderr={completed.stderr!r}" - ) from exc - return aggregate_stats( - entries, - read_errors(csv_prefix.with_name(f"{_CSV_PREFIX}_failures.csv")), - read_generator_warnings(completed.stderr), - ) diff --git a/tests/e2e/load/locustfile.py b/tests/e2e/load/locustfile.py deleted file mode 100644 index 4aa7517ca0b..00000000000 --- a/tests/e2e/load/locustfile.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -import os - -from locust import FastHttpUser, constant, task - -_MODEL = os.environ["LOAD_MODEL"] -_HEADERS = {"Authorization": f"Bearer {os.environ['LOAD_API_KEY']}"} -_PAYLOAD = { - "model": _MODEL, - "messages": [{"role": "user", "content": "load test ping"}], - "temperature": 0, - "max_tokens": 16, -} - - -class ChatUser(FastHttpUser): - wait_time = constant(0) - - @task - def chat(self) -> None: - self.client.post( # pyright: ignore[reportUnknownMemberType] # locust FastHttpSession.post types json/**kwargs as Any - "/chat/completions", - json=_PAYLOAD, - headers=_HEADERS, - name="/chat/completions", - ) diff --git a/tests/e2e/load/test_chat_completions_throughput_e2e.py b/tests/e2e/load/test_chat_completions_throughput_e2e.py deleted file mode 100644 index f36069c6999..00000000000 --- a/tests/e2e/load/test_chat_completions_throughput_e2e.py +++ /dev/null @@ -1,86 +0,0 @@ -import pytest - -from e2e_config import ( - LOAD_BASELINE_SECONDS, - LOAD_DURATION_SECONDS, - LOAD_MAX_FAILURE_RATIO, - LOAD_MAX_SERIAL_LATENCY_SECONDS, - LOAD_MIN_CONCURRENCY_EFFICIENCY, - LOAD_SPAWN_RATE, - LOAD_USERS, - PROXY_BASE_URL, -) -from load_client import LoadClient -from load_constants import LOAD_MODEL -from locust_load import run_chat_load - -pytestmark = [pytest.mark.e2e, pytest.mark.load] - - -class TestChatCompletionsThroughput: - @pytest.mark.skip( - reason=( - "LIT-5119: stage refuses most of the closed-loop load at the ELB (65.9% 502/503 on the " - "2026-08-02 run) because it idles at ~1 warm gateway replica; the per-replica SLO cannot " - "get a clean read until the fleet is pre-scaled for the load phase" - ) - ) - @pytest.mark.covers("reliability.perf.throughput.under_slo") - def test_sustains_throughput_slo_under_load(self, client: LoadClient, load_key: str) -> None: - baseline = run_chat_load( - base_url=PROXY_BASE_URL, - api_key=load_key, - model=LOAD_MODEL, - users=1, - spawn_rate=1, - duration_seconds=LOAD_BASELINE_SECONDS, - ) - - assert baseline.requests > 0 and baseline.requests_per_second > 0, ( - f"no requests completed against {PROXY_BASE_URL} in the {LOAD_BASELINE_SECONDS}s serial " - f"baseline; the load generator never drove traffic (proxy unreachable or model unservable). " - f"{baseline.diagnosis()}" - ) - assert baseline.failure_ratio <= LOAD_MAX_FAILURE_RATIO, ( - f"{baseline.failures}/{baseline.requests} requests failed in the serial baseline " - f"({baseline.failure_ratio:.1%} > {LOAD_MAX_FAILURE_RATIO:.1%} allowed), so it calibrates " - f"nothing: a request that fails in milliseconds reads as a fast one, passing the latency " - f"budget and inflating the floor it derives. {baseline.diagnosis()}" - ) - assert baseline.median_response_seconds <= LOAD_MAX_SERIAL_LATENCY_SECONDS, ( - f"one user with no competing load saw a median of {baseline.median_response_seconds * 1000:.0f}ms " - f"per request, over the {LOAD_MAX_SERIAL_LATENCY_SECONDS * 1000:.0f}ms budget; the request path " - f"itself is slow, before any concurrency. {baseline.diagnosis()}" - ) - - replica_rps = baseline.requests_per_second - min_rps = replica_rps * LOAD_MIN_CONCURRENCY_EFFICIENCY - - result = run_chat_load( - base_url=PROXY_BASE_URL, - api_key=load_key, - model=LOAD_MODEL, - users=LOAD_USERS, - spawn_rate=LOAD_SPAWN_RATE, - duration_seconds=LOAD_DURATION_SECONDS, - ) - - assert result.requests > 0, ( - f"no requests completed against {PROXY_BASE_URL} in {LOAD_DURATION_SECONDS}s; " - f"the load generator never drove traffic (proxy unreachable or model unservable). " - f"{result.diagnosis()}" - ) - assert result.failure_ratio <= LOAD_MAX_FAILURE_RATIO, ( - f"{result.failures}/{result.requests} requests failed " - f"({result.failure_ratio:.1%} > {LOAD_MAX_FAILURE_RATIO:.1%} allowed); " - f"throughput of {result.requests_per_second:.1f} RPS is not a clean read under this error rate, " - f"since closed-loop throughput rises when requests fail fast. {result.diagnosis()}" - ) - assert result.requests_per_second >= min_rps, ( - f"sustained {result.requests_per_second:.1f} RPS over {LOAD_DURATION_SECONDS}s with " - f"{LOAD_USERS} users, below the {min_rps:.1f} RPS floor that one replica set on its own " - f"({replica_rps:.1f} RPS serial at a {baseline.median_response_seconds * 1000:.0f}ms median, " - f"x {LOAD_MIN_CONCURRENCY_EFFICIENCY:.0%}); concurrency made the request path slower per " - f"request than a single user did, so the fleet is serialising rather than merely saturated. " - f"{result.diagnosis()}" - )