diff --git a/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts new file mode 100644 index 00000000000..8dd5571f7af --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/settings/routerSettings.spec.ts @@ -0,0 +1,102 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { Role, users } from "../../fixtures/users"; + +const PRIMARY = "fake-openai-gpt-4"; +const FALLBACK = "fake-anthropic-claude"; + +/** + * Wipe any fallbacks for the primary model so the test is idempotent across + * retries and local reruns (the proxy persists router_settings to the DB). + */ +async function clearFallbackForPrimary(request: import("@playwright/test").APIRequestContext) { + const masterKey = users[Role.ProxyAdmin].password; + const auth = { Authorization: `Bearer ${masterKey}` }; + + const current = await request.get("http://localhost:4000/get/config/callbacks", { headers: auth }); + if (!current.ok()) return; + const body = await current.json(); + const router = body?.router_settings ?? {}; + const existing: Array> = Array.isArray(router.fallbacks) ? router.fallbacks : []; + const next = existing.filter((entry) => !(entry && PRIMARY in entry)); + if (next.length === existing.length) return; + + await request.post("http://localhost:4000/config/update", { + headers: auth, + data: { router_settings: { ...router, fallbacks: next } }, + }); +} + +test.describe("Router Settings - Fallbacks", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test.beforeEach(async ({ request }) => { + await clearFallbackForPrimary(request); + }); + + test.afterEach(async ({ request }) => { + await clearFallbackForPrimary(request); + }); + + test("Add a fallback and verify it appears in the table", async ({ page }) => { + await navigateToPage(page, Page.RouterSettings); + + // Four tabs: Loadbalancing / Routing Groups / Fallbacks / General — click Fallbacks + await page.getByRole("tab", { name: "Fallbacks" }).click(); + + // The model options come from /model_group/info, which AddFallbacks + // fires only after the modal mounts. Wait for that response so the + // dropdown is populated before we try to pick from it — without this + // the test races on CI (local SLOWMO masks the gap). + const modelsLoaded = page.waitForResponse( + (res) => res.url().includes("/model_group/info") && res.status() === 200, + { timeout: 15_000 }, + ); + await page.getByRole("button", { name: /Add Fallbacks/i }).click(); + await modelsLoaded; + + const modal = page.locator(".ant-modal:visible"); + await expect(modal).toBeVisible({ timeout: 5_000 }); + + // FallbackGroupConfig.tsx renders both selects with `showSearch`. The + // most stable interaction is: click to open + focus, type the model name to + // narrow the listbox to a single highlighted option, then press Enter. + // Verify each selection landed by watching the dialog's own state transition + // (the tab title updates to the picked primary; the fallback chain list + // populates) rather than by asserting on the dropdown popup, which sits in + // a custom getPopupContainer and is awkward to scope reliably. + const primarySelect = modal.locator(".ant-select").filter({ hasText: "Select primary model" }); + await primarySelect.click(); + await page.keyboard.type(PRIMARY); + await page.keyboard.press("Enter"); + await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({ timeout: 10_000 }); + + const fallbackSelect = modal.locator(".ant-select").filter({ hasText: "Select fallback models" }); + await fallbackSelect.click(); + await page.keyboard.type(FALLBACK); + await page.keyboard.press("Enter"); + await page.keyboard.press("Escape"); + // The Fallback Chain helper text reads "(N/10 used)"; once it ticks to 1 the + // selection has been recorded. + await expect(modal.getByText("(1/10 used)")).toBeVisible({ timeout: 10_000 }); + + // Save + await modal.getByRole("button", { name: /Save All Configurations/i }).click(); + + // Success toast + await expect(page.getByText(/fallback configuration\(s\) added successfully/i).first()) + .toBeVisible({ timeout: 10_000 }); + + // Modal closes, and a single row contains BOTH the primary and the fallback + // model — stronger than asserting each name appears somewhere in tbody, + // which could be satisfied by leftover rows from prior runs. + await expect(modal).not.toBeVisible({ timeout: 5_000 }); + + const newRow = page.locator("table tbody tr") + .filter({ hasText: PRIMARY }) + .filter({ hasText: FALLBACK }); + await expect(newRow).toHaveCount(1, { timeout: 10_000 }); + }); +});