test(e2e): perf run measures a steady phase and then the rolling-upgrade window on its own budget

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

tests/e2e/perf/perf_rig.py

@@ -38,6 +38,7 @@ class LoadShape:
     spawn_rate: float
     warmup_seconds: float
     measured_seconds: float
+    roll_seconds: float = 0.0
     processes: int = 1
 
 
@@ -54,6 +55,15 @@ class PerfRun:
         )
 
 
+@dataclass(frozen=True, slots=True)
+class PerfPhases:
+    """`steady` is the stack as deployed, left alone. `roll` is the window after it in which the lane
+    upgrades the stack under the same load (None when `LoadShape.roll_seconds` is 0, as on a dev box)."""
+
+    steady: PerfRun
+    roll: PerfRun | None
+
+
 @dataclass(frozen=True, slots=True)
 class PerfBudget:
     """Flat limits a measured phase must hold on its own, with no earlier run to compare against."""
@@ -148,7 +158,7 @@ class PerfRig:
             self.resources.defer(lambda key=key: self.proxy.delete_key(key))
         return MockedGroup(name=name, keys=keys)
 
-    def measure(self, group: MockedGroup, endpoints: tuple[str, ...], shape: LoadShape) -> PerfRun:
+    def measure(self, group: MockedGroup, endpoints: tuple[str, ...], shape: LoadShape) -> PerfPhases:
         def drive(seconds: float) -> LoadResult:
             return run_gateway_load(
                 base_url=PROXY_BASE_URL,
@@ -164,9 +174,16 @@ class PerfRig:
         warmup: Final = drive(shape.warmup_seconds)
         assert warmup.requests > 0, f"warm-up sent nothing: {warmup.diagnosis()}"
         before: Final = _sample_workers(self.proxy)
-        load: Final = drive(shape.measured_seconds)
-        after: Final = _sample_workers(self.proxy)
-        return PerfRun(load=load, rss_growth_mb=_worst_worker_growth_mb(before, after))
+        steady_load: Final = drive(shape.measured_seconds)
+        after_steady: Final = _sample_workers(self.proxy)
+        steady: Final = PerfRun(load=steady_load, rss_growth_mb=_worst_worker_growth_mb(before, after_steady))
+        if shape.roll_seconds <= 0:
+            return PerfPhases(steady=steady, roll=None)
+        roll_load: Final = drive(shape.roll_seconds)
+        after_roll: Final = _sample_workers(self.proxy)
+        return PerfPhases(
+            steady=steady, roll=PerfRun(load=roll_load, rss_growth_mb=_worst_worker_growth_mb(after_steady, after_roll))
+        )
 
     def problems(self, label: str, run: PerfRun, budget: PerfBudget) -> tuple[str, ...]:
         """Everything wrong with the run, empty when it held the budget, printed so the build log carries the numbers."""

tests/e2e/perf/test_1k_rps.py

@@ -1,6 +1,8 @@
 """Live perf: the gateway serves 1k requests per second over mocked deployments of every major provider
 that answer with provider-like delays, with zero failures, latency ceilings set from those delays plus a
-gateway overhead allowance, and no worker growing its RSS past a flat ceiling.
+gateway overhead allowance, and no worker growing its RSS past a flat ceiling. With E2E_PERF_ROLL_SECONDS set
+the same load keeps running through a second window in which the CI lane upgrades the stack to the
+candidate, and that window carries its own, looser budget.
 
 Deselected unless E2E_PERF_TESTS is set.
 """
@@ -39,6 +41,7 @@ SHAPE: Final = LoadShape(
     spawn_rate=float(os.environ.get("E2E_PERF_SPAWN_RATE", "100")),
     warmup_seconds=float(os.environ.get("E2E_PERF_WARMUP_SECONDS", "15")),
     measured_seconds=float(os.environ.get("E2E_PERF_MEASURED_SECONDS", "60")),
+    roll_seconds=float(os.environ.get("E2E_PERF_ROLL_SECONDS", "0")),
     processes=int(os.environ.get("E2E_PERF_LOCUST_PROCESSES", "4")),
 )
 BUDGET: Final = PerfBudget(
@@ -48,11 +51,21 @@ BUDGET: Final = PerfBudget(
     p99_ceiling_seconds=max(MOCK_DELAYS_SECONDS) + 2 * GATEWAY_OVERHEAD_ALLOWANCE_SECONDS,
     rss_growth_ceiling_mb=150.0,
 )
+ROLL_BUDGET: Final = PerfBudget(
+    min_requests_per_second=0.8 * TARGET_RPS,
+    p50_ceiling_seconds=BUDGET.p50_ceiling_seconds,
+    p90_ceiling_seconds=BUDGET.p99_ceiling_seconds,
+    p99_ceiling_seconds=BUDGET.p99_ceiling_seconds + 2.0,
+    rss_growth_ceiling_mb=BUDGET.rss_growth_ceiling_mb,
+)
 
 
 class TestThousandRps:
     def test_gateway_holds_1k_rps_over_mocked_providers(self, perf_rig: PerfRig) -> None:
         group: Final = perf_rig.mocked_group(PROVIDER_MODELS, MOCK_DELAYS_SECONDS)
-        run: Final = perf_rig.measure(group, ENDPOINTS, SHAPE)
-        problems: Final = perf_rig.problems("1k rps", run, BUDGET)
-        assert not problems, "; ".join((*problems, run.report()))
+        phases: Final = perf_rig.measure(group, ENDPOINTS, SHAPE)
+        steady_problems: Final = perf_rig.problems("1k rps steady", phases.steady, BUDGET)
+        roll_problems: Final = (
+            perf_rig.problems("1k rps under the rolling upgrade", phases.roll, ROLL_BUDGET) if phases.roll else ()
+        )
+        assert not (*steady_problems, *roll_problems), "; ".join((*steady_problems, *roll_problems))