From c4c831562066c00befe6fec1766ca1236beb7451 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:05:05 +0000 Subject: [PATCH] test(e2e): perf rig (mocked group, Locust phases, per-worker RSS, PerfBudget) and a 1k rps perf test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/AGENTS.md | 2 +- tests/e2e/gateway/perf_ci_config.yml | 11 ++ tests/e2e/perf/conftest.py | 7 ++ tests/e2e/perf/locust_load.py | 4 + tests/e2e/perf/perf_rig.py | 161 +++++++++++++++++++++++++ tests/e2e/perf/test_perf_1k_rps_e2e.py | 46 +++++++ 6 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/gateway/perf_ci_config.yml create mode 100644 tests/e2e/perf/perf_rig.py create mode 100644 tests/e2e/perf/test_perf_1k_rps_e2e.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 8b9132cf05a..76716b01ba0 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -44,7 +44,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) -- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in +- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). The perf suite proper is the `perf`-marked tests (`test_perf_*_e2e.py`), each a few lines on the `perf_rig` fixture (`perf_rig.py`): `mocked_group` registers a model group over `mock_response` deployments plus a key pool (zero provider spend, torn down by `ResourceManager`), `measure` runs Locust through a discarded warm-up and a kept measured phase and reads every worker's RSS from `/debug/memory/summary` before and after, and `check` asserts one `PerfBudget` (rps floor, p50/p90/p99 ceilings, worst-worker RSS growth ceiling, zero failures, Locust not saturated) and prints the numbers to the build log. The proxy for those runs boots from `gateway/perf_ci_config.yml` - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at, see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests diff --git a/tests/e2e/gateway/perf_ci_config.yml b/tests/e2e/gateway/perf_ci_config.yml new file mode 100644 index 00000000000..8e4fe811b0a --- /dev/null +++ b/tests/e2e/gateway/perf_ci_config.yml @@ -0,0 +1,11 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + store_model_in_db: true + +litellm_settings: + callbacks: ["prometheus"] + require_auth_for_metrics_endpoint: false + +router_settings: + num_retries: 0 + disable_cooldowns: true diff --git a/tests/e2e/perf/conftest.py b/tests/e2e/perf/conftest.py index e7659a547a4..d5f0da75bb3 100644 --- a/tests/e2e/perf/conftest.py +++ b/tests/e2e/perf/conftest.py @@ -2,10 +2,17 @@ from __future__ import annotations import pytest +from lifecycle import ResourceManager from load_client import LoadClient, build_client +from perf_rig import PerfRig from proxy_client import ProxyClient @pytest.fixture(scope="session") def client(proxy: ProxyClient) -> LoadClient: return build_client(proxy) + + +@pytest.fixture +def perf_rig(proxy: ProxyClient, resources: ResourceManager) -> PerfRig: + return PerfRig(proxy=proxy, resources=resources) diff --git a/tests/e2e/perf/locust_load.py b/tests/e2e/perf/locust_load.py index 6d0aa8e1159..b20d056cdab 100644 --- a/tests/e2e/perf/locust_load.py +++ b/tests/e2e/perf/locust_load.py @@ -191,12 +191,15 @@ def run_gateway_load( users: int, spawn_rate: float, duration_seconds: float, + processes: int = 1, ) -> LoadResult: """Drive `endpoints` from headless locust and aggregate what it reported. Each simulated user picks one of `api_keys`, so auth and budget lookups spread over a pool of virtual keys instead of keeping one key's cache entry permanently warm, and one of `endpoints` round robin, so the run covers every route the caller asked for. + `processes` above 1 has locust fork that many workers behind one master on this host, + which is how a single generator gets past the throughput one Python process can push. """ with tempfile.TemporaryDirectory(prefix="e2e-load-") as report_dir: csv_prefix = Path(report_dir) / _CSV_PREFIX @@ -221,6 +224,7 @@ def run_gateway_load( f"{int(duration_seconds)}s", "--exit-code-on-error", "0", + *(("--processes", str(processes)) if processes > 1 else ()), ], env={ **os.environ, diff --git a/tests/e2e/perf/perf_rig.py b/tests/e2e/perf/perf_rig.py new file mode 100644 index 00000000000..7b972ab0797 --- /dev/null +++ b/tests/e2e/perf/perf_rig.py @@ -0,0 +1,161 @@ +"""The setup every perf test runs on, so a test is a few lines naming what it loads and what it must hold. + +`PerfRig.mocked_group` registers one model group over deployments that answer with `mock_response`, so +the proxy runs auth, routing, logging and spend for every request while the provider call costs nothing, +plus a pool of keys scoped to it, all torn down by `ResourceManager`. `PerfRig.measure` drives Locust +through a warm-up phase that is discarded and a measured phase that is kept, reading every worker's RSS +over the network from /debug/memory/summary before and after so the proxy can run anywhere. `PerfRig.check` +asserts the run against one `PerfBudget` and prints the numbers so the build log carries them +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +from e2e_config import PROXY_BASE_URL, unique_marker +from lifecycle import ResourceManager +from locust_load import LoadResult, run_gateway_load +from memory_readings import RssReading, WorkerKey, read_rss_everywhere +from models import KeyGenerateBody, LiteLLMParamsBody +from phase_budget import AbsoluteBudget, Budget, violations +from proxy_client import ProxyClient + +RSS_SAMPLE_READS: Final = 32 + + +@dataclass(frozen=True, slots=True) +class MockedGroup: + name: str + keys: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class LoadShape: + users: int + spawn_rate: float + warmup_seconds: float + measured_seconds: float + processes: int = 1 + + +@dataclass(frozen=True, slots=True) +class PerfRun: + load: LoadResult + rss_growth_mb: float + + def report(self) -> str: + return ( + f"{self.load.requests} requests, {self.load.failures} failures, " + f"{self.load.requests_per_second:.0f} rps, {self.load.latency_summary()}, " + f"worst worker RSS growth {self.rss_growth_mb:.1f} MB; by endpoint: {self.load.endpoint_summary()}" + ) + + +@dataclass(frozen=True, slots=True) +class PerfBudget: + """Flat limits a measured phase must hold on its own, with no earlier run to compare against.""" + + min_requests_per_second: float + p50_ceiling_seconds: float + p90_ceiling_seconds: float + p99_ceiling_seconds: float + rss_growth_ceiling_mb: float + + def violations(self, run: PerfRun) -> tuple[str, ...]: + load: Final = run.load + ceilings: Final[tuple[Budget, ...]] = ( + AbsoluteBudget( + name="p50 latency", measured=load.p50_seconds, ceiling=self.p50_ceiling_seconds, unit="s", decimals=3 + ), + AbsoluteBudget( + name="p90 latency", measured=load.p90_seconds, ceiling=self.p90_ceiling_seconds, unit="s", decimals=3 + ), + AbsoluteBudget( + name="p99 latency", measured=load.p99_seconds, ceiling=self.p99_ceiling_seconds, unit="s", decimals=3 + ), + AbsoluteBudget( + name="worst worker RSS growth", + measured=run.rss_growth_mb, + ceiling=self.rss_growth_ceiling_mb, + unit=" MB", + ), + ) + throughput: Final = ( + () + if load.requests_per_second >= self.min_requests_per_second + else ( + f"throughput {load.requests_per_second:.0f} rps is under the {self.min_requests_per_second:.0f} rps floor", + ) + ) + return (*throughput, *violations(ceilings)) + + +def _mock_params(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody(model=model, api_key="sk-perf-not-used", mock_response="perf ok") + + +def _sample_workers(proxy: ProxyClient) -> dict[WorkerKey, RssReading]: + """One reading per worker. Each read of /debug/memory/summary lands on whichever worker took the + connection, so repeated reads are what covers all of them, keyed by replica, host and pid.""" + captures: Final = tuple(read_rss_everywhere(proxy) for _ in range(RSS_SAMPLE_READS)) + return { + reading.worker: reading for capture in captures for reading in capture.readings + } # comprehension-ok: flattening one level of readings + + +def _worst_worker_growth_mb(before: dict[WorkerKey, RssReading], after: dict[WorkerKey, RssReading]) -> float: + growths: Final = tuple( + after[worker].ram_usage_mb - before[worker].ram_usage_mb for worker in after if worker in before + ) + assert growths, ( + f"no worker answered /debug/memory/summary both before and after the load; before: {sorted(before)}, after: {sorted(after)}" + ) + return max(growths) + + +@dataclass(frozen=True, slots=True) +class PerfRig: + proxy: ProxyClient + resources: ResourceManager + + def mocked_group(self, models: tuple[str, ...], *, key_pool_size: int = 8) -> MockedGroup: + name: Final = f"perf-{unique_marker()}" + for model in models: + model_id = self.proxy.create_model(name, _mock_params(model)) + self.resources.defer(lambda model_id=model_id: self.proxy.delete_model(model_id)) + keys: Final = tuple( + self.proxy.generate_key(KeyGenerateBody(models=[name], key_alias=f"{name}-{index}")) + for index in range(key_pool_size) + ) + for key in keys: + self.resources.defer(lambda key=key: self.proxy.delete_key(key)) + return MockedGroup(name=name, keys=keys) + + def measure(self, group: MockedGroup, endpoints: tuple[str, ...], shape: LoadShape) -> PerfRun: + def drive(seconds: float) -> LoadResult: + return run_gateway_load( + base_url=PROXY_BASE_URL, + api_keys=group.keys, + model=group.name, + endpoints=endpoints, + users=shape.users, + spawn_rate=shape.spawn_rate, + duration_seconds=seconds, + processes=shape.processes, + ) + + warmup: Final = drive(shape.warmup_seconds) + assert warmup.requests > 0, f"warm-up sent nothing: {warmup.diagnosis()}" + before: Final = _sample_workers(self.proxy) + load: Final = drive(shape.measured_seconds) + after: Final = _sample_workers(self.proxy) + return PerfRun(load=load, rss_growth_mb=_worst_worker_growth_mb(before, after)) + + def check(self, label: str, run: PerfRun, budget: PerfBudget) -> None: + report: Final = run.report() + print(f"\n{label}: {report}") # noqa: T201 # the numbers the perf suite exists to report, read off the CI log + assert run.load.failures == 0, f"{run.load.failures} requests failed: {run.load.diagnosis()}; {report}" + assert not run.load.generator_warnings, f"locust saturated itself, run invalid: {run.load.generator_warnings}" + failed: Final = budget.violations(run) + assert not failed, "; ".join((*failed, report)) diff --git a/tests/e2e/perf/test_perf_1k_rps_e2e.py b/tests/e2e/perf/test_perf_1k_rps_e2e.py new file mode 100644 index 00000000000..efae5869ed0 --- /dev/null +++ b/tests/e2e/perf/test_perf_1k_rps_e2e.py @@ -0,0 +1,46 @@ +"""Live perf: the gateway serves 1k requests per second over mocked deployments of every major provider +with zero failures, flat latency ceilings and no worker growing its RSS past a flat ceiling. + +Deselected unless E2E_PERF_TESTS is set. +""" + +from __future__ import annotations + +import os +from typing import Final + +import pytest +from perf_rig import LoadShape, PerfBudget, PerfRig + +pytestmark: Final = [pytest.mark.e2e, pytest.mark.perf] + +PROVIDER_MODELS: Final = ( + "openai/gpt-5.1", + "anthropic/claude-sonnet-5", + "azure/gpt-5.1", + "bedrock/converse/us.anthropic.claude-sonnet-5", + "vertex_ai/gemini-3-pro-preview", + "gemini/gemini-3.1-flash-lite-preview", +) +ENDPOINTS: Final = ("/chat/completions", "/v1/messages") +SHAPE: Final = LoadShape( + users=int(os.environ.get("E2E_PERF_USERS", "200")), + spawn_rate=float(os.environ.get("E2E_PERF_SPAWN_RATE", "100")), + warmup_seconds=float(os.environ.get("E2E_PERF_WARMUP_SECONDS", "15")), + measured_seconds=float(os.environ.get("E2E_PERF_MEASURED_SECONDS", "60")), + processes=int(os.environ.get("E2E_PERF_LOCUST_PROCESSES", "4")), +) +BUDGET: Final = PerfBudget( + min_requests_per_second=float(os.environ.get("E2E_PERF_TARGET_RPS", "1000")), + p50_ceiling_seconds=1.0, + p90_ceiling_seconds=2.0, + p99_ceiling_seconds=3.0, + rss_growth_ceiling_mb=150.0, +) + + +class TestThousandRps: + def test_gateway_holds_1k_rps_over_mocked_providers(self, perf_rig: PerfRig) -> None: + group: Final = perf_rig.mocked_group(PROVIDER_MODELS) + run: Final = perf_rig.measure(group, ENDPOINTS, SHAPE) + perf_rig.check("1k rps", run, BUDGET)