Merge pull request #38862 from BerriAI/litellm_/litellm-e2e-rc-1-99-0-11cfd8

test(e2e): backport the select-anchoring and router-fallback spec de-flakes

tests/e2e/models.py

@@ -229,6 +229,7 @@ class ChatBody(BaseModel):
     reasoning_effort: str | None = None
     thinking: ThinkingParam | None = None
     service_tier: str | None = None
+    prompt_cache_key: str | None = None
     tools: Sequence[ChatTool | McpChatTool] | None = None
     tool_choice: str | None = None
     guardrails: list[str] | None = None

tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py

@@ -12,11 +12,16 @@ header is exercised with a real nonzero value instead of passing vacuously. The
 backend is gpt-5.5 because it reports cached tokens on the second call; the
 gpt-5.6 line reports cache writes and never a read, which would leave the
 cache-read header at zero forever. The raw-transport send is used because the
-typed chat client validates bodies and drops headers. OpenAI caching is
-best-effort, so the prime+measure round retries with a fresh prefix before
-failing.
+typed chat client validates bodies and drops headers.
+
+OpenAI publishes a primed prefix asynchronously and routes lookups by
+prompt_cache_key, so a measure fired the instant the prime returns can miss a
+prefix that is about to become readable. Each round pins a cache key and re-reads
+the prefix it already paid to prime before spending a fresh one.
 """
 
+import time
+
 import pytest
 
 from cost_rows import approx_equal, cacheable_prefix, register_priced_model
@@ -31,6 +36,8 @@ pytestmark = pytest.mark.e2e
 BACKEND = "openai/gpt-5.5"
 OPENAI_API_KEY = "os.environ/OPENAI_API_KEY"
 CACHE_ATTEMPTS = 3
+CACHE_REREADS = 3
+CACHE_SETTLE_SECONDS = 2.0
 
 INPUT_RATE = 4e-05
 OUTPUT_RATE = 8e-05
@@ -70,7 +77,7 @@ class TestCostHeaders:
             ),
         )
 
-        def priced_call(content: str) -> StreamingResponse:
+        def priced_call(content: str, cache_key: str) -> StreamingResponse:
             response = client.proxy.transport.send(
                 "/chat/completions",
                 headers=client.proxy.transport.bearer(scoped_key),
@@ -78,21 +85,30 @@ class TestCostHeaders:
                     model=model,
                     messages=[ChatMessage(role="user", content=content)],
                     max_completion_tokens=4000,
+                    prompt_cache_key=cache_key,
                 ),
             )
             assert response.ok, f"chat failed (status {response.status_code}): {response.body[:300]}"
             return response
 
-        for _ in range(CACHE_ATTEMPTS):
-            prefix = cacheable_prefix(unique_marker())
-            priced_call(f"{prefix}\nReply with the single word ready.")
-            measured = priced_call(f"{prefix}\nReply with the single word measured.")
-            if _header_cost(measured, "x-litellm-response-cost-cache-read") > 0:
-                break
-        else:
+        def prime_then_reread() -> StreamingResponse | None:
+            marker = unique_marker()
+            prefix = cacheable_prefix(marker)
+            priced_call(f"{prefix}\nReply with the single word ready.", marker)
+            for _ in range(CACHE_REREADS):
+                time.sleep(CACHE_SETTLE_SECONDS)
+                response = priced_call(f"{prefix}\nReply with the single word measured.", marker)
+                if _header_cost(response, "x-litellm-response-cost-cache-read") > 0:
+                    return response
+            return None
+
+        rounds = (prime_then_reread() for _ in range(CACHE_ATTEMPTS))
+        measured = next((response for response in rounds if response is not None), None)
+        if measured is None:
             pytest.fail(
-                f"no cache read landed across {CACHE_ATTEMPTS} prime+measure rounds; "
-                "the cache-read cost header was never exercised with a nonzero value"
+                f"no cache read landed across {CACHE_ATTEMPTS} prime rounds of "
+                f"{CACHE_REREADS} re-reads each; the cache-read cost header was never "
+                "exercised with a nonzero value"
             )
 
         total = measured.response_cost

tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts

@@ -1,4 +1,4 @@
-import { expect, test, type Page as PlaywrightPage } from "@playwright/test";
+import { expect, test, type Locator, type Page as PlaywrightPage } from "@playwright/test";
 import { ADMIN_STORAGE_PATH } from "../../constants";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
@@ -17,43 +17,49 @@ async function openTemplateSelect(page: PlaywrightPage) {
   return trigger;
 }
 
+function pollPixelsBelowTrigger(trigger: Locator, popup: Locator) {
+  return expect.poll(async () => {
+    const triggerBox = await trigger.boundingBox();
+    const popupBox = await popup.boundingBox();
+    if (!triggerBox || !popupBox) return null;
+    return popupBox.y - (triggerBox.y + triggerBox.height);
+  });
+}
+
+function pollPopupOverlapsTrigger(trigger: Locator, popup: Locator) {
+  return expect.poll(async () => {
+    const triggerBox = await trigger.boundingBox();
+    const popupBox = await popup.boundingBox();
+    if (!triggerBox || !popupBox) return null;
+    return popupBox.y < triggerBox.y + triggerBox.height && popupBox.y + popupBox.height > triggerBox.y;
+  });
+}
+
 test.describe("Auto Router template select anchoring", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
   test("opens the options below the trigger rather than over it", async ({ page }) => {
     await page.setViewportSize({ width: 1280, height: 900 });
     const trigger = await openTemplateSelect(page);
-    const triggerBox = await trigger.boundingBox();
 
     await trigger.click();
     const popup = page.locator('[data-slot="select-content"]');
     await expect(popup).toBeVisible();
-    const popupBox = await popup.boundingBox();
-
-    expect(triggerBox).not.toBeNull();
-    expect(popupBox).not.toBeNull();
 
     // Item-aligned mode reports "none" and puts the active item over the trigger.
     await expect(popup).toHaveAttribute("data-side", "bottom");
-    expect(popupBox!.y).toBeGreaterThanOrEqual(triggerBox!.y + triggerBox!.height);
+    await pollPixelsBelowTrigger(trigger, popup).toBeGreaterThanOrEqual(0);
   });
 
   test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => {
     await page.setViewportSize({ width: 1280, height: 560 });
     const trigger = await openTemplateSelect(page);
     await trigger.scrollIntoViewIfNeeded();
-    const triggerBox = await trigger.boundingBox();
 
     await trigger.click();
     const popup = page.locator('[data-slot="select-content"]');
     await expect(popup).toBeVisible();
-    const popupBox = await popup.boundingBox();
 
-    expect(triggerBox).not.toBeNull();
-    expect(popupBox).not.toBeNull();
-
-    const overlaps =
-      popupBox!.y < triggerBox!.y + triggerBox!.height && popupBox!.y + popupBox!.height > triggerBox!.y;
-    expect(overlaps).toBe(false);
+    await pollPopupOverlapsTrigger(trigger, popup).toBe(false);
   });
 });

tests/e2e/ui/tests/settings/routerSettings.spec.ts

@@ -117,6 +117,11 @@ const ADMIN_AUTH = {
   Authorization: `Bearer ${users[Role.ProxyAdmin].password}`,
 };
 
+// Five probes 2s apart outlast the e2e stack's proxy_config_reload_interval_seconds of 7.
+const SETTLE_INTERVAL_MS = 2_000;
+const SETTLE_PROBES = 5;
+const SETTLE_TIMEOUT_MS = 60_000;
+
 /**
  * Apply a router_settings patch through the typed /config/update contract. The
  * server merges it over existing settings (request wins), so only the passed keys
@@ -133,6 +138,21 @@ async function patchRouterSettings(
   expect(res.ok(), `seed /config/update failed: ${res.status()} ${await res.text()}`).toBeTruthy();
 }
 
+/**
+ * Spreads its samples across more than one reload cycle: a single reply only proves the one
+ * replica that served it has reloaded, not the sibling still on the pre-update config.
+ */
+async function sampleStatuses(probe: () => Promise<number>): Promise<readonly number[]> {
+  return Array.from({ length: SETTLE_PROBES }).reduce<Promise<readonly number[]>>(
+    async (taken, _unused, index) => {
+      const sofar = await taken;
+      if (index > 0) await new Promise((resolve) => setTimeout(resolve, SETTLE_INTERVAL_MS));
+      return [...sofar, await probe()];
+    },
+    Promise.resolve([]),
+  );
+}
+
 test.describe("Router Settings - Loadbalancing", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
@@ -252,28 +272,34 @@ test.describe("Router Settings - Fallbacks serve the request", () => {
   });
 
   test("a request to an unreachable model is answered by its fallback", async ({ page, request }) => {
-    const chat = async () =>
-      request.post("/v1/chat/completions", {
-        headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
-        data: {
-          model: BROKEN_PRIMARY,
-          messages: [{ role: "user", content: "fallback probe" }],
-        },
-      });
+    const chatStatus = async () =>
+      (
+        await request.post("/v1/chat/completions", {
+          headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
+          data: {
+            model: BROKEN_PRIMARY,
+            messages: [{ role: "user", content: "fallback probe" }],
+          },
+        })
+      ).status();
 
-    // The control: it proves the reply below could only have come from the fallback.
-    expect((await chat()).status(), "broken primary unexpectedly succeeded on its own").toBeGreaterThanOrEqual(400);
+    // The control: every replica must reject, or the reply below could have come from one
+    // that was still serving a fallback left behind by an earlier attempt.
+    await expect
+      .poll(async () => (await sampleStatuses(chatStatus)).every((status) => status >= 400), {
+        timeout: SETTLE_TIMEOUT_MS,
+        message: "broken primary unexpectedly succeeded on its own",
+      })
+      .toBe(true);
 
     await patchRouterSettings(request, {
       fallbacks: [{ [BROKEN_PRIMARY]: [PRIMARY] }],
     } as Partial<NonNullable<ConfigYAML["router_settings"]>>);
 
-    // Same call now succeeds, served by the fallback model.
+    // One success is the whole claim here, so this waits for a first sighting rather than
+    // for every replica: demanding a streak would also assert a fallback hit rate.
     await expect
-      .poll(async () => (await chat()).status(), {
-        timeout: 30_000,
-        message: "fallback never took effect",
-      })
+      .poll(chatStatus, { timeout: SETTLE_TIMEOUT_MS, message: "fallback never took effect" })
       .toBe(200);
 
     // And the playground renders a reply for a model whose own upstream is down.