From 40d17a499ff9fc87c1bb597ef499ca79ef8f1181 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:16:14 +0000 Subject: [PATCH] test(e2e): perf rig reports problems as values and the test asserts them Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/AGENTS.md | 2 +- tests/e2e/perf/perf_rig.py | 18 ++++++----- tests/e2e/perf/session_anomaly.py | 43 ++++++-------------------- tests/e2e/perf/test_perf_1k_rps_e2e.py | 3 +- 4 files changed, 23 insertions(+), 43 deletions(-) diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index 76716b01ba0..0860191bd3d 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -44,7 +44,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) -- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). The perf suite proper is the `perf`-marked tests (`test_perf_*_e2e.py`), each a few lines on the `perf_rig` fixture (`perf_rig.py`): `mocked_group` registers a model group over `mock_response` deployments plus a key pool (zero provider spend, torn down by `ResourceManager`), `measure` runs Locust through a discarded warm-up and a kept measured phase and reads every worker's RSS from `/debug/memory/summary` before and after, and `check` asserts one `PerfBudget` (rps floor, p50/p90/p99 ceilings, worst-worker RSS growth ceiling, zero failures, Locust not saturated) and prints the numbers to the build log. The proxy for those runs boots from `gateway/perf_ci_config.yml` +- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). The perf suite proper is the `perf`-marked tests (`test_perf_*_e2e.py`), each a few lines on the `perf_rig` fixture (`perf_rig.py`): `mocked_group` registers a model group over `mock_response` deployments plus a key pool (zero provider spend, torn down by `ResourceManager`), `measure` runs Locust through a discarded warm-up and a kept measured phase and reads every worker's RSS from `/debug/memory/summary` before and after, and `problems` returns what the run broke in one `PerfBudget` (rps floor, p50/p90/p99 ceilings, worst-worker RSS growth ceiling, zero failures, Locust not saturated), which the test asserts empty, and prints the numbers to the build log. The proxy for those runs boots from `gateway/perf_ci_config.yml` - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at, see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests diff --git a/tests/e2e/perf/perf_rig.py b/tests/e2e/perf/perf_rig.py index 7b972ab0797..7f9aae2d1e8 100644 --- a/tests/e2e/perf/perf_rig.py +++ b/tests/e2e/perf/perf_rig.py @@ -4,8 +4,8 @@ the proxy runs auth, routing, logging and spend for every request while the provider call costs nothing, plus a pool of keys scoped to it, all torn down by `ResourceManager`. `PerfRig.measure` drives Locust through a warm-up phase that is discarded and a measured phase that is kept, reading every worker's RSS -over the network from /debug/memory/summary before and after so the proxy can run anywhere. `PerfRig.check` -asserts the run against one `PerfBudget` and prints the numbers so the build log carries them +over the network from /debug/memory/summary before and after so the proxy can run anywhere. `PerfRig.problems` +reports what the run broke in one `PerfBudget` and prints the numbers so the build log carries them """ from __future__ import annotations @@ -152,10 +152,14 @@ class PerfRig: after: Final = _sample_workers(self.proxy) return PerfRun(load=load, rss_growth_mb=_worst_worker_growth_mb(before, after)) - def check(self, label: str, run: PerfRun, budget: PerfBudget) -> None: + def problems(self, label: str, run: PerfRun, budget: PerfBudget) -> tuple[str, ...]: + """Everything wrong with the run, empty when it held the budget, printed so the build log carries the numbers.""" report: Final = run.report() print(f"\n{label}: {report}") # noqa: T201 # the numbers the perf suite exists to report, read off the CI log - assert run.load.failures == 0, f"{run.load.failures} requests failed: {run.load.diagnosis()}; {report}" - assert not run.load.generator_warnings, f"locust saturated itself, run invalid: {run.load.generator_warnings}" - failed: Final = budget.violations(run) - assert not failed, "; ".join((*failed, report)) + failures: Final = (f"{run.load.failures} requests failed: {run.load.diagnosis()}",) if run.load.failures else () + saturated: Final = ( + (f"locust saturated itself, run invalid: {run.load.generator_warnings}",) + if run.load.generator_warnings + else () + ) + return (*failures, *saturated, *budget.violations(run)) diff --git a/tests/e2e/perf/session_anomaly.py b/tests/e2e/perf/session_anomaly.py index b68482e965f..e105b076943 100644 --- a/tests/e2e/perf/session_anomaly.py +++ b/tests/e2e/perf/session_anomaly.py @@ -70,37 +70,24 @@ class AnomalyReport: @property def warm_cache_read_share(self) -> float: - billed = ( - self.warm_uncached_input_tokens - + self.warm_cache_read_tokens - + self.warm_cache_creation_tokens - ) + billed = self.warm_uncached_input_tokens + self.warm_cache_read_tokens + self.warm_cache_creation_tokens return self.warm_cache_read_tokens / billed if billed else 0.0 def _system_prefix_block(marker: str) -> TextBlock: - text = " ".join( - f"Project context paragraph {index} for session {marker}." for index in range(300) - ) + text = " ".join(f"Project context paragraph {index} for session {marker}." for index in range(300)) return TextBlock(text=text, cache_control=CacheControl()) def _user_turn_text(marker: str, turn_index: int) -> str: - notes = " ".join( - f"Working note {index} of turn {turn_index} in session {marker}." - for index in range(80) - ) + notes = " ".join(f"Working note {index} of turn {turn_index} in session {marker}." for index in range(80)) return f"Reply with one short sentence.\n{notes}" def _reminder_turn() -> RichMessage: return RichMessage( role="system", - content=[ - TextBlock( - text="Keep the answer to one short sentence." - ) - ], + content=[TextBlock(text="Keep the answer to one short sentence.")], ) @@ -127,9 +114,7 @@ def retried( return retried(call, attempts - 1, backoff_seconds, sleep) -def _metric( - result: Result[SessionMessagesResponse], turn_index: int, latency_seconds: float -) -> TurnMetric: +def _metric(result: Result[SessionMessagesResponse], turn_index: int, latency_seconds: float) -> TurnMetric: if isinstance(result, Success): usage = result.data.usage return TurnMetric( @@ -167,11 +152,7 @@ def _drive_turns( return () user_turn = RichMessage( role="user", - content=[ - TextBlock( - text=_user_turn_text(marker, turn_index), cache_control=CacheControl() - ) - ], + content=[TextBlock(text=_user_turn_text(marker, turn_index), cache_control=CacheControl())], ) started = time.monotonic() result = retried( @@ -190,9 +171,7 @@ def _drive_turns( turn = _metric(result, turn_index, time.monotonic() - started) if not isinstance(result, Success): return (turn,) - assistant_turn = RichMessage( - role="assistant", content=[TextBlock(text=result.data.text or "Understood.")] - ) + assistant_turn = RichMessage(role="assistant", content=[TextBlock(text=result.data.text or "Understood.")]) return ( turn, *_drive_turns( @@ -242,9 +221,7 @@ def run_concurrent_sessions( ) -> tuple[TurnMetric, ...]: with ThreadPoolExecutor(max_workers=sessions) as pool: futures = [ - pool.submit( - run_session, transport, key, model, turns_per_session, attempts_per_turn - ) + pool.submit(run_session, transport, key, model, turns_per_session, attempts_per_turn) for _ in range(sessions) ] return tuple(turn for future in futures for turn in future.result()) @@ -296,7 +273,5 @@ def summarize(turns: tuple[TurnMetric, ...], planned_turns: int) -> AnomalyRepor warm_uncached_input_tokens=sum(turn.uncached_input_tokens for turn in warm), warm_cache_read_tokens=sum(turn.cache_read_tokens for turn in warm), warm_cache_creation_tokens=sum(turn.cache_creation_tokens for turn in warm), - p95_turn_seconds=_p95( - tuple(turn.latency_seconds for turn in turns if turn.ok) - ), + p95_turn_seconds=_p95(tuple(turn.latency_seconds for turn in turns if turn.ok)), ) diff --git a/tests/e2e/perf/test_perf_1k_rps_e2e.py b/tests/e2e/perf/test_perf_1k_rps_e2e.py index efae5869ed0..5847ac5a981 100644 --- a/tests/e2e/perf/test_perf_1k_rps_e2e.py +++ b/tests/e2e/perf/test_perf_1k_rps_e2e.py @@ -43,4 +43,5 @@ class TestThousandRps: def test_gateway_holds_1k_rps_over_mocked_providers(self, perf_rig: PerfRig) -> None: group: Final = perf_rig.mocked_group(PROVIDER_MODELS) run: Final = perf_rig.measure(group, ENDPOINTS, SHAPE) - perf_rig.check("1k rps", run, BUDGET) + problems: Final = perf_rig.problems("1k rps", run, BUDGET) + assert not problems, "; ".join((*problems, run.report()))