diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 79c1234e35f..c653ff96935 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -16,7 +16,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO. Marked `load` so it is collected last (see the parent conftest) and never perturbs latency-sensitive suites +- `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO; marked `load` so the parent conftest collects it last and it never perturbs latency-sensitive suites - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees, and does not use the shared transport harness diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a4f65523b64..48199c544c0 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -45,9 +45,6 @@ def pytest_configure(config: pytest.Config) -> None: def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: - """Run `load`-marked tests last. A load test saturates the shared proxy, so - letting it run before a latency- or rate-limit-sensitive suite would skew that - suite; ordering it last keeps a whole-tree `pytest tests/e2e` run honest.""" items.sort(key=lambda item: item.get_closest_marker("load") is not None) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index ff57d3c1e9b..2d0ad93e53d 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -61,12 +61,6 @@ POLL_TIMEOUT = float(os.environ.get("E2E_POLL_TIMEOUT", "120")) POLL_INTERVAL = float(os.environ.get("E2E_POLL_INTERVAL", "5")) REQUEST_TIMEOUT = float(os.environ.get("E2E_REQUEST_TIMEOUT", "60")) -# Locust throughput load test (tests/e2e/load). Defaults mirror the concurrency -# and pass criterion the load test is meant to guard; every knob is overridable -# so the same test can be pointed at any deployment (a beefy stage cluster wants -# more users and a higher RPS floor than a laptop compose stack). LOAD_MIN_RPS is -# the SLO the run must clear and LOAD_MAX_FAILURE_RATIO caps the share of requests -# allowed to error before the throughput number is considered meaningless. LOAD_USERS = int(os.environ.get("E2E_LOAD_USERS", "750")) LOAD_SPAWN_RATE = float(os.environ.get("E2E_LOAD_SPAWN_RATE", "50")) LOAD_DURATION_SECONDS = float(os.environ.get("E2E_LOAD_DURATION_SECONDS", "60")) diff --git a/tests/e2e/load/conftest.py b/tests/e2e/load/conftest.py index 9dfbed03470..5558cf84774 100644 --- a/tests/e2e/load/conftest.py +++ b/tests/e2e/load/conftest.py @@ -1,17 +1,3 @@ -"""Load suite's `client` fixture plus the mock deployment it hammers. - -The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker -live in the parent tests/e2e/conftest.py. LoadClient holds the shared Gateway, so -the `resources` fixture cleans up keys this suite creates. - -The throughput test drives thousands of requests, so it points them at a mock -deployment (litellm_params.mock_response) rather than a real provider: the proxy -short-circuits before any upstream call, so the run measures proxy overhead under -concurrency instead of a provider's latency, cost, or rate limit. The deployment -is registered via /model/new when the proxy does not already list it (compose has -it in static config; stage does not), mirroring the router suite. -""" - from __future__ import annotations from collections.abc import Iterator diff --git a/tests/e2e/load/load_client.py b/tests/e2e/load/load_client.py index dee43c09d42..df7c91fadf9 100644 --- a/tests/e2e/load/load_client.py +++ b/tests/e2e/load/load_client.py @@ -1,12 +1,3 @@ -"""Client for the throughput load test. - -The suite drives raw /chat/completions traffic through Locust (its own HTTP -client, the one exception to the shared-transport rule, since a load generator -must measure request throughput itself) and only uses the shared Gateway to -create the key and the mock deployment it hammers, so this client just carries -the Gateway the shared lifecycle needs for cleanup. -""" - from __future__ import annotations from dataclasses import dataclass diff --git a/tests/e2e/load/locust_load.py b/tests/e2e/load/locust_load.py index f32eb657f4c..79ecd8ddd36 100644 --- a/tests/e2e/load/locust_load.py +++ b/tests/e2e/load/locust_load.py @@ -1,19 +1,3 @@ -"""Drive a Locust throughput load at the proxy's /chat/completions and hand back -the aggregate stats a throughput SLO can be asserted on. - -Locust is run as a subprocess rather than embedded: it monkey-patches the stdlib -with gevent at import, which deadlocks pytest's own machinery if imported in-process, -so the generator lives in its own interpreter (see locustfile.py) and reports back -through its `--json` summary on stdout. It is also the one place in tests/e2e that -does not go through the shared transport - a load generator has to own its HTTP -client to measure throughput, and FastHttpUser is the client Locust ships for it. - -Traffic is a real POST /chat/completions with a real bearer key against the live -proxy, so the whole request path (auth, routing, logging, spend) runs under -production-like concurrency; only the upstream provider is a mock deployment, so -the number reflects proxy overhead rather than a provider's latency or rate limit. -""" - from __future__ import annotations import os diff --git a/tests/e2e/load/locustfile.py b/tests/e2e/load/locustfile.py index cef4e1b26c4..4aa7517ca0b 100644 --- a/tests/e2e/load/locustfile.py +++ b/tests/e2e/load/locustfile.py @@ -1,12 +1,3 @@ -"""Locust user for the throughput load test, run as a subprocess by locust_load.py. - -Reads the target model and bearer key from the environment (locust_load passes -LOAD_MODEL and LOAD_API_KEY; --host carries the proxy base URL) and hammers -/chat/completions with FastHttpUser, the high-throughput client Locust ships. The -model is a mock deployment, so every request exercises the full proxy path without -an upstream provider call. -""" - from __future__ import annotations import os diff --git a/tests/e2e/load/test_chat_completions_throughput_e2e.py b/tests/e2e/load/test_chat_completions_throughput_e2e.py index df0aef7ae28..e5323be1c85 100644 --- a/tests/e2e/load/test_chat_completions_throughput_e2e.py +++ b/tests/e2e/load/test_chat_completions_throughput_e2e.py @@ -1,17 +1,3 @@ -"""Live e2e: the proxy sustains its throughput SLO on /chat/completions under -concurrent load, so a regression that quietly slows the request path (a bad Redis -SSL config forcing a handshake per request, a dependency bump that adds per-request -overhead) fails here instead of in a customer's load test. - -CodSpeed benchmarks the SDK with no IO; this fills the gap it cannot see by driving -real concurrent HTTP through the whole stack (proxy + Postgres + Redis) with Locust. -Traffic hits a mock deployment (see conftest) so the number reflects proxy overhead, -not a provider's latency or rate limit. The defaults (users, duration, and the RPS -floor) come from e2e_config and are overridable per deployment; the assertion is the -aggregate throughput clearing LOAD_MIN_RPS with the failure share under -LOAD_MAX_FAILURE_RATIO, since a run that mostly errors makes the RPS meaningless. -""" - import pytest from e2e_config import (