diff --git a/tests/e2e/perf/perf_rig.py b/tests/e2e/perf/perf_rig.py index a2c82dc4c55..8e726689f7f 100644 --- a/tests/e2e/perf/perf_rig.py +++ b/tests/e2e/perf/perf_rig.py @@ -38,6 +38,7 @@ class LoadShape: spawn_rate: float warmup_seconds: float measured_seconds: float + roll_seconds: float = 0.0 processes: int = 1 @@ -54,6 +55,15 @@ class PerfRun: ) +@dataclass(frozen=True, slots=True) +class PerfPhases: + """`steady` is the stack as deployed, left alone. `roll` is the window after it in which the lane + upgrades the stack under the same load (None when `LoadShape.roll_seconds` is 0, as on a dev box).""" + + steady: PerfRun + roll: PerfRun | None + + @dataclass(frozen=True, slots=True) class PerfBudget: """Flat limits a measured phase must hold on its own, with no earlier run to compare against.""" @@ -148,7 +158,7 @@ class PerfRig: self.resources.defer(lambda key=key: self.proxy.delete_key(key)) return MockedGroup(name=name, keys=keys) - def measure(self, group: MockedGroup, endpoints: tuple[str, ...], shape: LoadShape) -> PerfRun: + def measure(self, group: MockedGroup, endpoints: tuple[str, ...], shape: LoadShape) -> PerfPhases: def drive(seconds: float) -> LoadResult: return run_gateway_load( base_url=PROXY_BASE_URL, @@ -164,9 +174,16 @@ class PerfRig: warmup: Final = drive(shape.warmup_seconds) assert warmup.requests > 0, f"warm-up sent nothing: {warmup.diagnosis()}" before: Final = _sample_workers(self.proxy) - load: Final = drive(shape.measured_seconds) - after: Final = _sample_workers(self.proxy) - return PerfRun(load=load, rss_growth_mb=_worst_worker_growth_mb(before, after)) + steady_load: Final = drive(shape.measured_seconds) + after_steady: Final = _sample_workers(self.proxy) + steady: Final = PerfRun(load=steady_load, rss_growth_mb=_worst_worker_growth_mb(before, after_steady)) + if shape.roll_seconds <= 0: + return PerfPhases(steady=steady, roll=None) + roll_load: Final = drive(shape.roll_seconds) + after_roll: Final = _sample_workers(self.proxy) + return PerfPhases( + steady=steady, roll=PerfRun(load=roll_load, rss_growth_mb=_worst_worker_growth_mb(after_steady, after_roll)) + ) def problems(self, label: str, run: PerfRun, budget: PerfBudget) -> tuple[str, ...]: """Everything wrong with the run, empty when it held the budget, printed so the build log carries the numbers.""" diff --git a/tests/e2e/perf/test_1k_rps.py b/tests/e2e/perf/test_1k_rps.py index 46b7ae920f9..87205756608 100644 --- a/tests/e2e/perf/test_1k_rps.py +++ b/tests/e2e/perf/test_1k_rps.py @@ -1,6 +1,8 @@ """Live perf: the gateway serves 1k requests per second over mocked deployments of every major provider that answer with provider-like delays, with zero failures, latency ceilings set from those delays plus a -gateway overhead allowance, and no worker growing its RSS past a flat ceiling. +gateway overhead allowance, and no worker growing its RSS past a flat ceiling. With E2E_PERF_ROLL_SECONDS set +the same load keeps running through a second window in which the CI lane upgrades the stack to the +candidate, and that window carries its own, looser budget. Deselected unless E2E_PERF_TESTS is set. """ @@ -39,6 +41,7 @@ SHAPE: Final = LoadShape( spawn_rate=float(os.environ.get("E2E_PERF_SPAWN_RATE", "100")), warmup_seconds=float(os.environ.get("E2E_PERF_WARMUP_SECONDS", "15")), measured_seconds=float(os.environ.get("E2E_PERF_MEASURED_SECONDS", "60")), + roll_seconds=float(os.environ.get("E2E_PERF_ROLL_SECONDS", "0")), processes=int(os.environ.get("E2E_PERF_LOCUST_PROCESSES", "4")), ) BUDGET: Final = PerfBudget( @@ -48,11 +51,21 @@ BUDGET: Final = PerfBudget( p99_ceiling_seconds=max(MOCK_DELAYS_SECONDS) + 2 * GATEWAY_OVERHEAD_ALLOWANCE_SECONDS, rss_growth_ceiling_mb=150.0, ) +ROLL_BUDGET: Final = PerfBudget( + min_requests_per_second=0.8 * TARGET_RPS, + p50_ceiling_seconds=BUDGET.p50_ceiling_seconds, + p90_ceiling_seconds=BUDGET.p99_ceiling_seconds, + p99_ceiling_seconds=BUDGET.p99_ceiling_seconds + 2.0, + rss_growth_ceiling_mb=BUDGET.rss_growth_ceiling_mb, +) class TestThousandRps: def test_gateway_holds_1k_rps_over_mocked_providers(self, perf_rig: PerfRig) -> None: group: Final = perf_rig.mocked_group(PROVIDER_MODELS, MOCK_DELAYS_SECONDS) - run: Final = perf_rig.measure(group, ENDPOINTS, SHAPE) - problems: Final = perf_rig.problems("1k rps", run, BUDGET) - assert not problems, "; ".join((*problems, run.report())) + phases: Final = perf_rig.measure(group, ENDPOINTS, SHAPE) + steady_problems: Final = perf_rig.problems("1k rps steady", phases.steady, BUDGET) + roll_problems: Final = ( + perf_rig.problems("1k rps under the rolling upgrade", phases.roll, ROLL_BUDGET) if phases.roll else () + ) + assert not (*steady_problems, *roll_problems), "; ".join((*steady_problems, *roll_problems))