test(e2e): route /router/settings to the control plane and keep the 429 and cooldown cells inside their windows

GET /router/settings is a management route, so the split transport now
sends it to the control plane instead of the data-plane gateway.

The rpm-1 key behind the 429 cells is spent right before the trip, after
the pair is registered, because the rate limiter's 60s window opens on
that request and the registrations' propagation waits could otherwise
outlast it. Recovery also accepts a 200 served by the benched deployment
itself, since its key's minute can be up by then.

The cooldown recovery deadline now counts from the last failure a stale
replica caused during propagation, because every failure re-arms the
cooldown TTL; the strict bench window stays anchored to the trip.

tests/e2e/router/reliability_support.py

@@ -20,7 +20,7 @@ from collections.abc import Sequence
 from pydantic import ValidationError
 
 from proxy_client import ProxyClient
-from e2e_config import PROXY_BASE_URL
+from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker
 from e2e_http import NetworkError, StreamHead, StreamingResponse
 from models import (
     CacheControl,
@@ -194,6 +194,18 @@ def create_always_rate_limited_deployment(
     )
 
 
+def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
+    """Uses up the one request an rpm_limit=1 key allows. The proxy's rate limiter
+    opens the key's 60s window on this call, so it goes right before the calls that
+    need the 429 and after the registrations, whose propagation waits could
+    otherwise eat the window."""
+    primed = chat_override(proxy, spent_key, CHEAP_OPENAI_MODEL, f"say hi {unique_marker()}")
+    assert primed.status_code == 200, (
+        f"the one request the rpm-limited key allows should have succeeded, got {primed.status_code}: "
+        f"{primed.body[:300]}"
+    )
+
+
 def create_zero_weight_backup_deployment(proxy: ProxyClient, name: str) -> str:
     """The other half of a failing pair: healthy, but weight 0, so the weighted shuffle
     never opens on it. It is reachable only once its sibling is benched and the

tests/e2e/router/test_reliability_cooldowns_e2e.py

@@ -15,20 +15,24 @@ after the trip every answer has to be either the deployment's own failure or a
 one replica has to have served from the backup by then. From then until shortly
 before the cooldown can lapse, every call has to land on the backup whichever
 replica takes it. Then the test polls until the weighted shuffle opens on the
-failing deployment again and the same failure comes back: that is the recovery,
-since a benched deployment is one the router will try again, not one it forgot.
+failing deployment again and the same failure comes back (or, for the 429 pair,
+its own 200 once the key's rpm window has reset): that is the recovery, since a
+benched deployment is one the router will try again, not one it forgot. Its
+deadline counts from the last failure a stale replica caused, because every
+failure re-arms the cooldown.
 
 The failures are the same real ones the retry tests use: a 1ms deadline and a
 bogus key on the real backend, and this proxy standing in as the upstream for
 the 500 (fronting a group whose only deployment is unreachable) and the 429
-(fronting a healthy group with a key that already spent its one request per
-minute).
+(fronting a healthy group with a key whose one request per minute is spent right
+before the trip, so its window outlasts the bench).
 """
 
 from __future__ import annotations
 
 import time
 from collections.abc import Iterator
+from dataclasses import dataclass
 
 import pytest
 from complexity_router_client import ComplexityRouterClient
@@ -46,6 +50,7 @@ from reliability_support import (
     create_bad_base_deployment,
     create_zero_weight_backup_deployment,
     model_id_of,
+    spend_only_request_of,
 )
 
 pytestmark = pytest.mark.e2e
@@ -91,23 +96,36 @@ def _backup_sighting(resp: StreamingResponse, elapsed: float, backup: str, failu
     return None
 
 
-def _seconds_until_first_backup(
+@dataclass(frozen=True, slots=True)
+class _Propagation:
+    first_backup_at: float
+    last_failure_at: float
+
+
+def _propagation_of(
     client: ComplexityRouterClient, key: str, group: str, backup: str, failure_status: int, tripped_at: float
-) -> float:
+) -> _Propagation:
     sightings = tuple(
-        _backup_sighting(resp, elapsed, backup, failure_status)
+        (elapsed, _backup_sighting(resp, elapsed, backup, failure_status))
         for elapsed, resp in _answers_while_replicas_catch_up(client, key, group, tripped_at)
     )
-    seen = tuple(elapsed for elapsed in sightings if elapsed is not None)
-    assert seen, (
+    backups = tuple(elapsed for elapsed, backup_at in sightings if backup_at is not None)
+    assert backups, (
         f"no replica served {group} from the backup within {REPLICA_PROPAGATION_SECONDS:.0f}s of the trip, so the "
         "cooldown never became visible"
     )
-    return seen[0]
+    return _Propagation(
+        first_backup_at=backups[0],
+        last_failure_at=max((elapsed for elapsed, backup_at in sightings if backup_at is None), default=0.0),
+    )
+
+
+def _reached_benched_deployment(resp: StreamingResponse, failing: str, failure_status: int) -> bool:
+    return resp.status_code == failure_status or model_id_of(resp) == failing
 
 
 def _assert_trips_then_recovers(
-    client: ComplexityRouterClient, key: str, group: str, backup: str, failure_status: int
+    client: ComplexityRouterClient, key: str, group: str, failing: str, backup: str, failure_status: int
 ) -> None:
     tripped_at = time.monotonic()
     tripped = _call_without_retries(client, key, group)
@@ -116,7 +134,7 @@ def _assert_trips_then_recovers(
         f"{tripped.body[:300]}"
     )
 
-    visible_after = _seconds_until_first_backup(client, key, group, backup, failure_status, tripped_at)
+    propagation = _propagation_of(client, key, group, backup, failure_status, tripped_at)
 
     bench_until = tripped_at + COOLDOWN_SECONDS - BENCH_MARGIN_SECONDS
     while time.monotonic() < bench_until:
@@ -124,17 +142,17 @@ def _assert_trips_then_recovers(
             _call_without_retries(client, key, group),
             backup,
             f"{time.monotonic() - tripped_at:.1f}s into a {COOLDOWN_SECONDS:.0f}s cooldown that became visible "
-            f"after {visible_after:.1f}s,",
+            f"after {propagation.first_backup_at:.1f}s,",
         )
 
-    recovery_deadline = tripped_at + COOLDOWN_SECONDS + RECOVERY_GRACE_SECONDS
+    recovery_deadline = tripped_at + propagation.last_failure_at + COOLDOWN_SECONDS + RECOVERY_GRACE_SECONDS
     while time.monotonic() < recovery_deadline:
         time.sleep(1)
-        if _call_without_retries(client, key, group).status_code == failure_status:
+        if _reached_benched_deployment(_call_without_retries(client, key, group), failing, failure_status):
             return
     pytest.fail(
         f"{group} never sent traffic back to its benched deployment within "
-        f"{COOLDOWN_SECONDS + RECOVERY_GRACE_SECONDS:.0f}s of the trip, so the cooldown never lapsed"
+        f"{COOLDOWN_SECONDS + RECOVERY_GRACE_SECONDS:.0f}s of its last failure, so the cooldown never lapsed"
     )
 
 
@@ -155,7 +173,7 @@ class TestReliabilityCooldowns:
         backup = create_zero_weight_backup_deployment(client.proxy, group)
         resources.defer(lambda: client.proxy.delete_model(backup))
 
-        _assert_trips_then_recovers(client, scoped_key, group, backup, failure_status=500)
+        _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=500)
 
     @pytest.mark.covers("reliability.cooldown.429.trips_then_recovers")
     def test_429_trips_cooldown_then_recovers(
@@ -165,11 +183,6 @@ class TestReliabilityCooldowns:
             KeyGenerateBody(models=[CHEAP_OPENAI_MODEL], rpm_limit=1, user_id="e2e-test-user")
         )
         resources.defer(lambda: client.proxy.delete_key(spent_key))
-        primed = chat_override(client.proxy, spent_key, CHEAP_OPENAI_MODEL, f"say hi {unique_marker()}")
-        assert primed.status_code == 200, (
-            f"the one request the rpm-limited key allows should have succeeded, got {primed.status_code}: "
-            f"{primed.body[:300]}"
-        )
 
         group = f"reliability-cooldown-429-{unique_marker()}"
         failing = create_always_rate_limited_deployment(
@@ -179,7 +192,8 @@ class TestReliabilityCooldowns:
         backup = create_zero_weight_backup_deployment(client.proxy, group)
         resources.defer(lambda: client.proxy.delete_model(backup))
 
-        _assert_trips_then_recovers(client, scoped_key, group, backup, failure_status=429)
+        spend_only_request_of(client.proxy, spent_key)
+        _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=429)
 
     @pytest.mark.covers("reliability.cooldown.auth.trips_then_recovers")
     def test_auth_failure_trips_cooldown_then_recovers(
@@ -191,7 +205,7 @@ class TestReliabilityCooldowns:
         backup = create_zero_weight_backup_deployment(client.proxy, group)
         resources.defer(lambda: client.proxy.delete_model(backup))
 
-        _assert_trips_then_recovers(client, scoped_key, group, backup, failure_status=401)
+        _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=401)
 
     @pytest.mark.covers("reliability.cooldown.timeout.trips_then_recovers")
     def test_timeout_trips_cooldown_then_recovers(
@@ -203,4 +217,4 @@ class TestReliabilityCooldowns:
         backup = create_zero_weight_backup_deployment(client.proxy, group)
         resources.defer(lambda: client.proxy.delete_model(backup))
 
-        _assert_trips_then_recovers(client, scoped_key, group, backup, failure_status=408)
+        _assert_trips_then_recovers(client, scoped_key, group, failing, backup, failure_status=408)

tests/e2e/router/test_reliability_retries_e2e.py

@@ -42,6 +42,7 @@ from reliability_support import (
     create_bad_base_deployment,
     create_zero_weight_backup_deployment,
     finish_reason_of,
+    spend_only_request_of,
 )
 
 pytestmark = pytest.mark.e2e
@@ -115,11 +116,6 @@ class TestReliabilityRetries:
             KeyGenerateBody(models=[CHEAP_OPENAI_MODEL], rpm_limit=1, user_id="e2e-test-user")
         )
         resources.defer(lambda: client.proxy.delete_key(spent_key))
-        primed = chat_override(client.proxy, spent_key, CHEAP_OPENAI_MODEL, f"say hi {unique_marker()}")
-        assert primed.status_code == 200, (
-            f"the one request the rpm-limited key allows should have succeeded, got {primed.status_code}: "
-            f"{primed.body[:300]}"
-        )
 
         group = f"reliability-retry-429-{unique_marker()}"
         failing = create_always_rate_limited_deployment(client.proxy, group, CHEAP_OPENAI_MODEL, spent_key)
@@ -127,6 +123,7 @@ class TestReliabilityRetries:
         backup = create_zero_weight_backup_deployment(client.proxy, group)
         resources.defer(lambda: client.proxy.delete_model(backup))
 
+        spend_only_request_of(client.proxy, spent_key)
         _assert_served_after_retry(_retry_once(client, scoped_key, group))
 
     @pytest.mark.covers("reliability.retry.auth.succeeds_within_retries")

tests/e2e/transport.py

@@ -302,6 +302,7 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = (
     "/global",
     "/config",
     "/guardrails",
+    "/router/settings",
     "/openapi.json",
 )