test(e2e): de-flake the cost-header cache read and the router fallback control

Two e2e tests fail on timing rather than on litellm behaviour. Measured over the
last ~35 litellm-e2e / litellm-e2e-ui runs:

  routerSettings.spec.ts:254  9/35 runs (7 flaky-on-retry, 2 hard failures)
  test_cost_headers_e2e.py    1/29 runs it appeared in

Router fallback control
-----------------------
The e2e stack runs replicaCount 2 with proxy_config_reload_interval_seconds 7,
and every request is routed independently, so an observation of the new config
only proves the replica that served it reloaded. patchRouterSettings returns as
soon as /config/update returns, and clearBrokenFallback never waits at all, so a
retry's one-shot control assertion could be answered by a sibling replica still
holding the previous attempt's fallback. That is exactly the observed pair of
errors: "fallback never took effect" on the first attempt and "broken primary
unexpectedly succeeded on its own" on the retry.

Both assertions now poll for a consecutive streak spanning more than one reload
cycle, mirroring the PROPAGATION_TIMEOUT / settle_propagation doctrine the Python
suite already applies in e2e_config.py.

Cost-header cache read
----------------------
The prime and measure calls fired back to back with no gap, and each retry threw
away the prefix it had just paid to prime in favour of a fresh one. OpenAI
publishes a primed prefix asynchronously and routes cache lookups by
prompt_cache_key, so the test was rerolling the least likely path to a hit.

Each round now pins a prompt_cache_key and re-reads the same primed prefix up to
CACHE_REREADS times before rotating, so a fresh prefix is spent only after the
primed one has genuinely failed to become readable.

No production code changes; prompt_cache_key is added to the e2e ChatBody model,
which serializes exclude_none and so is inert for every other caller.

tests/e2e/models.py

@@ -256,6 +256,7 @@ class ChatBody(BaseModel):
     reasoning_effort: str | None = None
     thinking: ThinkingParam | None = None
     service_tier: str | None = None
+    prompt_cache_key: str | None = None
     tools: Sequence[ChatTool | McpChatTool] | None = None
     tool_choice: str | None = None
     guardrails: list[str] | None = None

tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py

@@ -12,11 +12,16 @@ header is exercised with a real nonzero value instead of passing vacuously. The
 backend is gpt-5.5 because it reports cached tokens on the second call; the
 gpt-5.6 line reports cache writes and never a read, which would leave the
 cache-read header at zero forever. The raw-transport send is used because the
-typed chat client validates bodies and drops headers. OpenAI caching is
-best-effort, so the prime+measure round retries with a fresh prefix before
-failing.
+typed chat client validates bodies and drops headers.
+
+OpenAI publishes a primed prefix asynchronously and routes lookups by
+prompt_cache_key, so a measure fired the instant the prime returns can miss a
+prefix that is about to become readable. Each round pins a cache key and re-reads
+the prefix it already paid to prime before spending a fresh one.
 """
 
+import time
+
 import pytest
 
 from cost_rows import approx_equal, cacheable_prefix, register_priced_model
@@ -31,6 +36,8 @@ pytestmark = pytest.mark.e2e
 BACKEND = "openai/gpt-5.5"
 OPENAI_API_KEY = "os.environ/OPENAI_API_KEY"
 CACHE_ATTEMPTS = 3
+CACHE_REREADS = 3
+CACHE_SETTLE_SECONDS = 2.0
 
 INPUT_RATE = 4e-05
 OUTPUT_RATE = 8e-05
@@ -70,7 +77,7 @@ class TestCostHeaders:
             ),
         )
 
-        def priced_call(content: str) -> StreamingResponse:
+        def priced_call(content: str, cache_key: str) -> StreamingResponse:
             response = client.proxy.transport.send(
                 "/chat/completions",
                 headers=client.proxy.transport.bearer(scoped_key),
@@ -78,21 +85,33 @@ class TestCostHeaders:
                     model=model,
                     messages=[ChatMessage(role="user", content=content)],
                     max_completion_tokens=4000,
+                    prompt_cache_key=cache_key,
                 ),
             )
             assert response.ok, f"chat failed (status {response.status_code}): {response.body[:300]}"
             return response
 
+        def prime_then_reread() -> StreamingResponse | None:
+            marker = unique_marker()
+            prefix = cacheable_prefix(marker)
+            priced_call(f"{prefix}\nReply with the single word ready.", marker)
+            for _ in range(CACHE_REREADS):
+                time.sleep(CACHE_SETTLE_SECONDS)
+                response = priced_call(f"{prefix}\nReply with the single word measured.", marker)
+                if _header_cost(response, "x-litellm-response-cost-cache-read") > 0:
+                    return response
+            return None
+
+        measured: StreamingResponse | None = None
         for _ in range(CACHE_ATTEMPTS):
-            prefix = cacheable_prefix(unique_marker())
-            priced_call(f"{prefix}\nReply with the single word ready.")
-            measured = priced_call(f"{prefix}\nReply with the single word measured.")
-            if _header_cost(measured, "x-litellm-response-cost-cache-read") > 0:
+            measured = prime_then_reread()
+            if measured is not None:
                 break
-        else:
+        if measured is None:
             pytest.fail(
-                f"no cache read landed across {CACHE_ATTEMPTS} prime+measure rounds; "
-                "the cache-read cost header was never exercised with a nonzero value"
+                f"no cache read landed across {CACHE_ATTEMPTS} prime rounds of "
+                f"{CACHE_REREADS} re-reads each; the cache-read cost header was never "
+                "exercised with a nonzero value"
             )
 
         total = measured.response_cost

tests/e2e/ui/tests/settings/routerSettings.spec.ts

@@ -117,6 +117,11 @@ const ADMIN_AUTH = {
   Authorization: `Bearer ${users[Role.ProxyAdmin].password}`,
 };
 
+// Five probes 2s apart outlast the e2e stack's proxy_config_reload_interval_seconds of 7.
+const SETTLE_INTERVAL_MS = 2_000;
+const SETTLE_PROBES = 5;
+const SETTLE_TIMEOUT_MS = 60_000;
+
 /**
  * Apply a router_settings patch through the typed /config/update contract. The
  * server merges it over existing settings (request wins), so only the passed keys
@@ -133,6 +138,27 @@ async function patchRouterSettings(
   expect(res.ok(), `seed /config/update failed: ${res.status()} ${await res.text()}`).toBeTruthy();
 }
 
+/**
+ * Requires a consecutive streak because a single reply only proves the one replica that
+ * served it has reloaded, not the sibling still answering from the pre-update config.
+ */
+async function pollUntilSettled(
+  probe: () => Promise<number>,
+  matches: (status: number) => boolean,
+  message: string,
+): Promise<void> {
+  let streak = 0;
+  await expect
+    .poll(
+      async () => {
+        streak = matches(await probe()) ? streak + 1 : 0;
+        return streak;
+      },
+      { timeout: SETTLE_TIMEOUT_MS, intervals: [SETTLE_INTERVAL_MS], message },
+    )
+    .toBeGreaterThanOrEqual(SETTLE_PROBES);
+}
+
 test.describe("Router Settings - Loadbalancing", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
@@ -252,29 +278,30 @@ test.describe("Router Settings - Fallbacks serve the request", () => {
   });
 
   test("a request to an unreachable model is answered by its fallback", async ({ page, request }) => {
-    const chat = async () =>
-      request.post("/v1/chat/completions", {
-        headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
-        data: {
-          model: BROKEN_PRIMARY,
-          messages: [{ role: "user", content: "fallback probe" }],
-        },
-      });
+    const chatStatus = async () =>
+      (
+        await request.post("/v1/chat/completions", {
+          headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
+          data: {
+            model: BROKEN_PRIMARY,
+            messages: [{ role: "user", content: "fallback probe" }],
+          },
+        })
+      ).status();
 
     // The control: it proves the reply below could only have come from the fallback.
-    expect((await chat()).status(), "broken primary unexpectedly succeeded on its own").toBeGreaterThanOrEqual(400);
+    await pollUntilSettled(
+      chatStatus,
+      (status) => status >= 400,
+      "broken primary unexpectedly succeeded on its own",
+    );
 
     await patchRouterSettings(request, {
       fallbacks: [{ [BROKEN_PRIMARY]: [PRIMARY] }],
     } as Partial<NonNullable<ConfigYAML["router_settings"]>>);
 
     // Same call now succeeds, served by the fallback model.
-    await expect
-      .poll(async () => (await chat()).status(), {
-        timeout: 30_000,
-        message: "fallback never took effect",
-      })
-      .toBe(200);
+    await pollUntilSettled(chatStatus, (status) => status === 200, "fallback never took effect");
 
     // And the playground renders a reply for a model whose own upstream is down.
     await openPlayground(page);