diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index ed57dd71759..419cfd6dedd 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -18,7 +18,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) -- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint and budgeting p50/p90/p99 latency, RSS, CPU-per-request, and log-bytes-per-request as ratios against the same run's healthy phase; needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic +- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set, driven weekly by `.github/workflows/test-e2e-redis-chaos.yml`), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke diff --git a/tests/e2e/load/phase_budget.py b/tests/e2e/load/phase_budget.py index 7d354a678bc..066e2579da8 100644 --- a/tests/e2e/load/phase_budget.py +++ b/tests/e2e/load/phase_budget.py @@ -1,18 +1,27 @@ """Comparing one load phase against another, for tests that degrade a dependency mid-run. -A chaos phase's absolute numbers say very little on their own: RSS scales with worker count, -latency with core count, so a ceiling calibrated on one machine is meaningless on the next. -What travels is the ratio against a healthy phase measured on the same machine in the same run. +Two shapes of ceiling, because the metrics divide into two kinds. RSS and CPU are +machine-shaped: RSS scales with worker count and CPU with core count, so an absolute number +calibrated on one runner means nothing on the next, and what travels is the ratio against a +healthy phase measured on the same machine in the same run. Latency and log volume are not: +a ratio there is actively misleading, because a dependency that fails fast once its breaker +opens can make the degraded phase look cheaper than the healthy one while still being far +slower or noisier than a user should ever see. Those get a flat ceiling, which is the promise +the test is actually making. """ from __future__ import annotations from dataclasses import dataclass -from typing import Final +from typing import Final, TypeAlias + + +def _rendered(value: float, unit: str, decimals: int) -> str: + return f"{value:.{decimals}f}{unit}" @dataclass(frozen=True, slots=True) -class Budget: +class RatioBudget: """One metric's healthy value, its degraded value, and how much growth is allowed.""" name: str @@ -27,26 +36,46 @@ class Budget: """How many times the baseline the degraded value is, or None if there is no baseline.""" return self.degraded / self.baseline if self.baseline > 0 else None - def _rendered(self, value: float) -> str: - return f"{value:.{self.decimals}f}{self.unit}" - def violation(self) -> str | None: """Why this metric fails its budget, or None if it passes.""" ratio: Final = self.ratio if ratio is None: return ( - f"{self.name} measured {self._rendered(self.baseline)} in the healthy phase, so there is nothing " - f"to compare the degraded phase against; the measurement did not happen" + f"{self.name} measured {_rendered(self.baseline, self.unit, self.decimals)} in the healthy phase, " + f"so there is nothing to compare the degraded phase against; the measurement did not happen" ) if ratio > self.ratio_ceiling: return ( - f"{self.name} went from {self._rendered(self.baseline)} healthy to " - f"{self._rendered(self.degraded)} degraded, {ratio:.1f}x the baseline and past the " - f"{self.ratio_ceiling:.1f}x allowed" + f"{self.name} went from {_rendered(self.baseline, self.unit, self.decimals)} healthy to " + f"{_rendered(self.degraded, self.unit, self.decimals)} degraded, {ratio:.1f}x the baseline and past " + f"the {self.ratio_ceiling:.1f}x allowed" ) return None +@dataclass(frozen=True, slots=True) +class AbsoluteBudget: + """One metric's degraded value against a flat ceiling, for metrics a ratio cannot bound.""" + + name: str + measured: float + ceiling: float + unit: str + decimals: int = 1 + + def violation(self) -> str | None: + """Why this metric fails its budget, or None if it passes.""" + if self.measured > self.ceiling: + return ( + f"{self.name} measured {_rendered(self.measured, self.unit, self.decimals)} in the degraded phase, " + f"past the {_rendered(self.ceiling, self.unit, self.decimals)} allowed" + ) + return None + + +Budget: TypeAlias = RatioBudget | AbsoluteBudget + + def violations(budgets: tuple[Budget, ...]) -> tuple[str, ...]: """Every budget the run blew, so one failure reports all of them instead of the first.""" return tuple(violation for budget in budgets if (violation := budget.violation()) is not None) diff --git a/tests/e2e/load/test_phase_budget.py b/tests/e2e/load/test_phase_budget.py index 2355fb7cf4c..ea9e56afb0d 100644 --- a/tests/e2e/load/test_phase_budget.py +++ b/tests/e2e/load/test_phase_budget.py @@ -2,14 +2,16 @@ from __future__ import annotations from typing import Final -from phase_budget import Budget, violations +from phase_budget import AbsoluteBudget, RatioBudget, violations -def _budget(*, baseline: float, degraded: float, ceiling: float = 2.0) -> Budget: - return Budget(name="p99 RSS", baseline=baseline, degraded=degraded, ratio_ceiling=ceiling, unit=" MB", decimals=0) +def _budget(*, baseline: float, degraded: float, ceiling: float = 2.0) -> RatioBudget: + return RatioBudget( + name="p99 RSS", baseline=baseline, degraded=degraded, ratio_ceiling=ceiling, unit=" MB", decimals=0 + ) -class TestBudget: +class TestRatioBudget: def test_growth_within_the_ceiling_is_not_a_violation(self) -> None: assert _budget(baseline=100, degraded=199).violation() is None @@ -37,7 +39,7 @@ class TestBudget: assert "nothing to compare" in violation def test_the_unit_and_decimals_carry_into_the_message(self) -> None: - violation: Final = Budget( + violation: Final = RatioBudget( name="p99 latency", baseline=0.16, degraded=9.5, ratio_ceiling=8.0, unit="s", decimals=3 ).violation() @@ -46,13 +48,40 @@ class TestBudget: assert "9.500s" in violation +class TestAbsoluteBudget: + def test_a_value_under_the_ceiling_is_not_a_violation(self) -> None: + assert AbsoluteBudget(name="p99 latency", measured=1.2, ceiling=5.0, unit="s", decimals=3).violation() is None + + def test_a_value_exactly_at_the_ceiling_is_allowed(self) -> None: + assert AbsoluteBudget(name="p99 latency", measured=5.0, ceiling=5.0, unit="s", decimals=3).violation() is None + + def test_a_value_past_the_ceiling_reports_the_measurement_and_the_ceiling(self) -> None: + violation: Final = AbsoluteBudget( + name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3 + ).violation() + + assert violation is not None + assert "9.500s" in violation + assert "5.000s allowed" in violation + + def test_a_flat_ceiling_fails_a_degraded_phase_that_is_cheaper_than_its_baseline(self) -> None: + # The whole reason this shape exists: once the breaker opens, requests skip Redis instead + # of waiting on its socket timeout, so the chaos phase can measure faster than the healthy + # one. A ratio against that baseline passes; the user still waited 9.5s. + assert _budget(baseline=20.0, degraded=9.5, ceiling=2.0).violation() is None + assert AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s").violation() is not None + + def test_a_zero_measurement_is_not_a_violation(self) -> None: + assert AbsoluteBudget(name="log bytes per request", measured=0, ceiling=12_000, unit=" B").violation() is None + + class TestViolations: def test_every_blown_budget_is_reported_not_just_the_first(self) -> None: blown: Final = violations( ( _budget(baseline=100, degraded=500), _budget(baseline=100, degraded=120), - Budget(name="CPU per request", baseline=10, degraded=90, ratio_ceiling=6.0, unit=" ms"), + RatioBudget(name="CPU per request", baseline=10, degraded=90, ratio_ceiling=6.0, unit=" ms"), ) ) @@ -60,5 +89,17 @@ class TestViolations: assert blown[0].startswith("p99 RSS") assert blown[1].startswith("CPU per request") + def test_both_budget_shapes_report_together(self) -> None: + blown: Final = violations( + ( + _budget(baseline=100, degraded=500), + AbsoluteBudget(name="p99 latency", measured=9.5, ceiling=5.0, unit="s", decimals=3), + ) + ) + + assert len(blown) == 2 + assert blown[0].startswith("p99 RSS") + assert blown[1].startswith("p99 latency") + def test_a_run_inside_every_budget_reports_nothing(self) -> None: assert violations((_budget(baseline=100, degraded=150),)) == () diff --git a/tests/e2e/load/test_redis_chaos_e2e.py b/tests/e2e/load/test_redis_chaos_e2e.py index 153be656566..94c879d29cb 100644 --- a/tests/e2e/load/test_redis_chaos_e2e.py +++ b/tests/e2e/load/test_redis_chaos_e2e.py @@ -50,7 +50,7 @@ from lifecycle import ResourceManager from load_client import LoadClient from locust_load import LoadResult, run_gateway_load from models import KeyGenerateBody, LiteLLMParamsBody -from phase_budget import Budget, violations +from phase_budget import AbsoluteBudget, Budget, RatioBudget, violations from proxy_client import ProxyClient from proxy_usage import ProxyUsageSampler, UsageWindow @@ -70,23 +70,25 @@ BASELINE_SECONDS: Final = 60.0 CHAOS_SECONDS: Final = 90.0 REDIS_PAUSE_MS: Final = int(CHAOS_SECONDS * 1000) -# Chaos-phase ceilings, as a multiple of the same metric in the baseline phase. Ratios rather -# than absolutes because every absolute here is machine-shaped: RSS scales with worker count -# and latency with core count, so a number calibrated on one runner means nothing on another. -# Calibrated from local runs under CLIENT PAUSE ALL that came in around 4x latency at every -# percentile, 1.03x RSS and 4.4x CPU per request, and deliberately loose: the regression these -# guard against grew memory by an order of magnitude, so catching it does not need a tight -# bound, and a tight one would flake on a shared CI runner. Latency gets the most slack because -# it is the metric a Redis outage is legitimately allowed to move, by its socket timeout on -# every call a request attempts. -CHAOS_LATENCY_RATIO_CEILING: Final = 12.0 +# RSS and CPU are budgeted as a multiple of the same metric in the baseline phase, because both +# are machine-shaped: RSS scales with worker count and CPU with core count, so a number +# calibrated on one runner means nothing on another. Calibrated from local runs under CLIENT +# PAUSE ALL that came in around 1.03x RSS and 4.4x CPU per request, and deliberately loose: the +# regression these guard against grew memory by an order of magnitude, so catching it does not +# need a tight bound, and a tight one would flake on a shared CI runner. CHAOS_RSS_RATIO_CEILING: Final = 1.5 CHAOS_CPU_PER_REQUEST_RATIO_CEILING: Final = 6.0 -# Uncalibrated: no chaos run has measured this yet, since JSON_LOGS and the padded payload -# landed after the last run this file's other ceilings were calibrated from. Deliberately loose -# until a real run tightens it; the failed-tracking alert body that motivates this test already -# logs the full request metadata per timeout, so a JSON-encoded traceback storm should dwarf this. -CHAOS_LOG_BYTES_PER_REQUEST_RATIO_CEILING: Final = 20.0 + +# Latency and log volume get flat ceilings instead, because a ratio cannot bound either one. Once +# the breaker opens, a request skips Redis rather than waiting on its socket timeout, so the chaos +# phase can come in faster than baseline (local runs measured p90 at 0.61x) and a ratio passes on a +# phase that was never slow. What a user actually cares about is the wall-clock number, which these +# hold directly. Calibrated from local runs whose worst chaos phase was p50 0.66s, p90 0.74s, p99 +# 1.20s and 3.4 KB of log per request, then left roughly 3x loose for a shared CI runner. +CHAOS_P50_LATENCY_CEILING_SECONDS: Final = 2.0 +CHAOS_P90_LATENCY_CEILING_SECONDS: Final = 3.0 +CHAOS_P99_LATENCY_CEILING_SECONDS: Final = 5.0 +CHAOS_LOG_BYTES_PER_REQUEST_CEILING: Final = 12_000.0 DRAIN_TIMEOUT_SECONDS: Final = 30.0 DRAIN_POLL_SECONDS: Final = 1.0 @@ -289,19 +291,12 @@ def _drive(keys: tuple[str, ...], seconds: float) -> LoadResult: ) -def _latency_budget(percentile: str, baseline: float, degraded: float) -> Budget: - return Budget( - name=f"{percentile} latency", - baseline=baseline, - degraded=degraded, - ratio_ceiling=CHAOS_LATENCY_RATIO_CEILING, - unit="s", - decimals=3, - ) +def _latency_budget(percentile: str, measured: float, ceiling: float) -> Budget: + return AbsoluteBudget(name=f"{percentile} latency", measured=measured, ceiling=ceiling, unit="s", decimals=3) def _rss_budget(percentile: str, baseline: UsageWindow, degraded: UsageWindow, fraction: float) -> Budget: - return Budget( + return RatioBudget( name=f"{percentile} RSS", baseline=baseline.rss_percentile(fraction) / 2**20, degraded=degraded.rss_percentile(fraction) / 2**20, @@ -312,18 +307,18 @@ def _rss_budget(percentile: str, baseline: UsageWindow, degraded: UsageWindow, f def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: - """What a Redis outage is allowed to cost, measured against the same run's healthy phase. + """What a Redis outage is allowed to cost. Every request still succeeding is the headline assertion, but a proxy can answer every request while leaking: the v1.100.0 regression (LIT-6780) served traffic the whole way up - to a 61 GB worker. These bound the cost of serving it. + to a 61 GB worker. These bound the cost of serving it. RSS and CPU are bounded against the + same run's healthy phase, latency and log bytes against a flat ceiling; see phase_budget + for why the two kinds of metric cannot share one shape. Latency and RSS are budgeted at p50, p90 and p99 so a regression that only shows up in the - tail (or only in the median) cannot hide behind the other. Latency gets the loosest bound - because a timing-out Redis legitimately adds its socket_timeout to every request that - touches it, several times over on a retried request. RSS gets the tightest: the failure - path has no business allocating more per request. CPU and log bytes are each budgeted once, - as an amount per request rather than per percentile: cores-busy saturates at the worker + tail (or only in the median) cannot hide behind the other. RSS gets the tightest bound: the + failure path has no business allocating more per request. CPU and log bytes are each budgeted + once, as an amount per request rather than per percentile: cores-busy saturates at the worker count under load, so its percentiles read the same whether a request costs 10 ms of CPU or 40, and cannot budget anything; per-request is the figure that actually moves. Log bytes isolates the cost of the failed-tracking alert's own noisy error handling from the CPU it @@ -331,24 +326,23 @@ def _chaos_budgets(baseline: Phase, chaos: Phase) -> tuple[Budget, ...]: CPU number. """ return ( - _latency_budget("p50", baseline.load.p50_seconds, chaos.load.p50_seconds), - _latency_budget("p90", baseline.load.p90_seconds, chaos.load.p90_seconds), - _latency_budget("p99", baseline.load.p99_seconds, chaos.load.p99_seconds), + _latency_budget("p50", chaos.load.p50_seconds, CHAOS_P50_LATENCY_CEILING_SECONDS), + _latency_budget("p90", chaos.load.p90_seconds, CHAOS_P90_LATENCY_CEILING_SECONDS), + _latency_budget("p99", chaos.load.p99_seconds, CHAOS_P99_LATENCY_CEILING_SECONDS), _rss_budget("p50", baseline.usage, chaos.usage, 0.5), _rss_budget("p90", baseline.usage, chaos.usage, 0.9), _rss_budget("p99", baseline.usage, chaos.usage, 0.99), - Budget( + RatioBudget( name="CPU per request", baseline=baseline.cpu_seconds_per_request * 1000, degraded=chaos.cpu_seconds_per_request * 1000, ratio_ceiling=CHAOS_CPU_PER_REQUEST_RATIO_CEILING, unit=" ms", ), - Budget( + AbsoluteBudget( name="log bytes per request", - baseline=baseline.log_bytes_per_request, - degraded=chaos.log_bytes_per_request, - ratio_ceiling=CHAOS_LOG_BYTES_PER_REQUEST_RATIO_CEILING, + measured=chaos.log_bytes_per_request, + ceiling=CHAOS_LOG_BYTES_PER_REQUEST_CEILING, unit=" B", decimals=0, ), @@ -447,8 +441,7 @@ class TestRedisChaos: blown: Final = violations(_chaos_budgets(baseline, chaos)) assert not blown, ( - f"pausing Redis cost the proxy more than the socket timeout on the calls it attempts: " - f"{'; '.join(blown)}. {report}" + f"pausing Redis cost the proxy more than a Redis outage is allowed to: {'; '.join(blown)}. {report}" ) rows: Final = proxy.poll_logs_for_key(keys[0], min_rows=1)