Merge pull request #40026 from BerriAI/litellm_fix_migrated_pages_sidebar_test

test(ui): make navigation smoke resilient to router refactors
This commit is contained in:
yuneng-jiang 2026-09-07 11:17:32 -07:00 committed by GitHub
commit 6dfcc46c9e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 282 additions and 148 deletions

View file

@ -1,50 +1,142 @@
/**
* Source of truth for the App Router migration E2E suites.
*
* Add an entry (legacy sidebar page id -> route segment) once a page's migration
* has MERGED to the branch under test. Consumers pick it up automatically:
* - migration smoke (tests/migration/migratedPages.spec.ts), via MIGRATED_E2E_SEGMENTS:
* default mount: npm run e2e:migration
* server-root-path mount: SERVER_ROOT_PATH=/<root> npm run e2e:migration:root
* - navigation specs that assert per-page URLs (tests/navigation/sidebar.spec.ts)
*
* Keep this in lockstep with MIGRATED_PAGES in src/utils/migratedPages.ts.
*/
export const MIGRATED_E2E_PAGES: Record<string, string> = {
"api-keys": "api-keys",
models: "models-and-endpoints",
api_ref: "api-reference",
"llm-playground": "playground",
projects: "projects",
"access-groups": "access-groups",
budgets: "budgets",
workflows: "workflows",
"guardrails-monitor": "guardrails-monitor",
"mcp-servers": "mcp-servers",
"search-tools": "search-tools",
"tag-management": "tag-management",
"vector-stores": "vector-stores",
memory: "memory",
policies: "policies",
guardrails: "guardrails",
prompts: "prompts",
"tool-policies": "tool-policies",
skills: "skills",
caching: "caching",
"cost-tracking": "cost-tracking",
"transform-request": "transform-request",
"ui-theme": "ui-theme",
logs: "logs",
"admin-panel": "admin-panel",
"logging-and-alerts": "logging-and-alerts",
"model-hub-table": "model-hub-table",
new_usage: "usage",
usage: "old-usage",
agents: "agents",
"router-settings": "router-settings",
users: "users",
teams: "teams",
organizations: "organizations",
};
export type MigratedPage = Readonly<{
segment: string;
linkName: string | RegExp;
group?: string;
content: Readonly<{ role: "heading" | "tab" | "button"; name: string }> | Readonly<{ text: string }>;
unlicensedText?: string;
}>;
export const MIGRATED_E2E_SEGMENTS: string[] = [...new Set(Object.values(MIGRATED_E2E_PAGES))];
export const MIGRATED_E2E_PAGES: Readonly<Record<string, MigratedPage>> = {
"api-keys": { segment: "api-keys", linkName: "Virtual Keys", content: { role: "heading", name: "Virtual Keys" } },
models: {
segment: "models-and-endpoints",
linkName: "Models + Endpoints",
content: { role: "heading", name: "Model Management" },
},
api_ref: {
segment: "api-reference",
linkName: "API Reference",
content: { role: "heading", name: "OpenAI Compatible Proxy: API Reference" },
},
"llm-playground": { segment: "playground", linkName: "Playground", content: { role: "tab", name: "Chat" } },
projects: {
segment: "projects",
linkName: /^Projects(?: Beta)?$/,
content: { role: "heading", name: "Projects" },
},
"access-groups": {
segment: "access-groups",
linkName: "Access Groups",
content: { role: "heading", name: "Access Groups" },
},
budgets: { segment: "budgets", linkName: "Budgets", content: { role: "heading", name: "Budgets" } },
workflows: {
segment: "workflows",
linkName: "Workflow Runs",
group: "Agentic",
content: { text: "Workflow Runs" },
},
"guardrails-monitor": {
segment: "guardrails-monitor",
linkName: "Guardrails Monitor",
content: { role: "heading", name: "Guardrails Monitor" },
},
"mcp-servers": {
segment: "mcp-servers",
linkName: "MCP Servers",
content: { role: "heading", name: "MCP Servers" },
},
"search-tools": {
segment: "search-tools",
linkName: "Search Tools",
group: "Tools",
content: { role: "heading", name: "Search Tools" },
},
"tag-management": {
segment: "tag-management",
linkName: "Tag Management",
group: "Experimental",
content: { role: "heading", name: "Tag Management" },
},
"vector-stores": {
segment: "vector-stores",
linkName: "Vector Stores",
group: "Tools",
content: { role: "heading", name: "Vector Store Management" },
},
memory: { segment: "memory", linkName: "Memory", group: "Agentic", content: { role: "heading", name: "Memory" } },
policies: { segment: "policies", linkName: "Policies", content: { role: "tab", name: "Policy Simulator" } },
guardrails: { segment: "guardrails", linkName: "Guardrails", content: { role: "tab", name: "Guardrails" } },
prompts: {
segment: "prompts",
linkName: "Prompts",
group: "Experimental",
content: { role: "button", name: "Add New Prompt" },
},
"tool-policies": {
segment: "tool-policies",
linkName: "Tool Policies",
group: "Tools",
content: { role: "heading", name: "Tool Policies" },
},
skills: { segment: "skills", linkName: "Skills", content: { role: "heading", name: "Skills" } },
caching: { segment: "caching", linkName: "Response Cache", content: { role: "tab", name: "Cache Settings" } },
"cost-tracking": {
segment: "cost-tracking",
linkName: "Cost Tracking",
group: "Settings",
content: { text: "Cost Tracking Settings" },
},
"transform-request": {
segment: "transform-request",
linkName: "API Playground",
group: "Experimental",
content: { role: "heading", name: "Playground" },
},
"ui-theme": {
segment: "ui-theme",
linkName: "UI Theme",
group: "Settings",
content: { role: "heading", name: "UI Theme Customization" },
},
logs: { segment: "logs", linkName: "Logs", content: { role: "heading", name: "Request Logs" } },
"admin-panel": {
segment: "admin-panel",
linkName: "Admin Settings",
group: "Settings",
content: { role: "heading", name: "Admin Access" },
},
"logging-and-alerts": {
segment: "logging-and-alerts",
linkName: "Logging & Alerts",
group: "Settings",
content: { role: "tab", name: "Logging Callbacks" },
},
"model-hub-table": {
segment: "model-hub-table",
linkName: "AI Hub",
content: { role: "heading", name: "AI Hub" },
},
new_usage: { segment: "usage", linkName: "Usage", content: { role: "heading", name: "Usage View" } },
usage: {
segment: "old-usage",
linkName: "Old Usage",
group: "Experimental",
content: { role: "tab", name: "All Up" },
},
agents: { segment: "agents", linkName: "Agents", group: "Agentic", content: { role: "heading", name: "Agents" } },
"router-settings": {
segment: "router-settings",
linkName: "Router Settings",
group: "Settings",
content: { role: "heading", name: "Routing Settings" },
},
users: { segment: "users", linkName: "Internal Users", content: { role: "tab", name: "Users" } },
teams: { segment: "teams", linkName: "Teams", content: { role: "heading", name: "Teams" } },
organizations: {
segment: "organizations",
linkName: "Organizations",
content: { text: "Click on an organization ID to view its details." },
unlicensedText: "This is a LiteLLM Enterprise feature, and requires a valid key to use. Get a trial key here.",
},
};

View file

@ -1,5 +1,29 @@
import { Page } from "../fixtures/pages";
import { Page as PlaywrightPage, expect } from "@playwright/test";
import { UI_BASE_URL } from "../constants";
export const sidebarLink = (page: PlaywrightPage, name: string | RegExp) =>
page.getByRole("complementary").getByRole("link", { name, exact: true });
export async function clickSidebarLink(page: PlaywrightPage, name: string | RegExp, groupName?: string): Promise<void> {
const link = sidebarLink(page, name);
if (groupName && !(await link.isVisible())) {
const group = page.getByRole("complementary").getByRole("button", { name: groupName, exact: true });
await expect(group).toBeVisible();
if ((await group.getAttribute("aria-expanded")) === "false") {
await group.click();
}
}
await link.click();
}
export async function expectUiRoute(page: PlaywrightPage, segment: string): Promise<void> {
const root = (process.env.SERVER_ROOT_PATH ?? "").replace(/\/+$/, "");
const expected = new URL(`${root}/ui/${segment}`, UI_BASE_URL);
await expect(page, `navigate to ${expected.pathname}`).toHaveURL(
(url) => url.origin === expected.origin && url.pathname.replace(/\/+$/, "") === expected.pathname,
);
}
/**
* Navigates to a specific page using the page query parameter.

View file

@ -1,17 +1,25 @@
# App Router migration smoke
A growing E2E smoke for pages migrated from the legacy `?page=` switch to App
Router path routes. For each migrated page it clicks the page's sidebar link, checks
the URL is the path route and the page renders, reloads it, then clicks off to a
legacy page and back to confirm navigation still works. It runs in two situations:
the default mount and a non-root `SERVER_ROOT_PATH` mount.
Router path routes. For each page it clicks the sidebar link by its accessible
name, verifies the destination's content, reloads it, then visits Virtual Keys
and returns. It runs at the default mount and a non-root `SERVER_ROOT_PATH` mount
Link selection does not depend on `href` formatting. URL assertions compare the
origin and pathname, allowing a trailing slash, query string, and fragment while
rejecting another route or mount. Reloads must return a successful document,
and each journey must finish without uncaught browser errors
## Adding a page
When a page's migration merges, add its route segment to
`tests/e2e/ui/fixtures/migratedPages.ts` (keep it in lockstep with `MIGRATED_PAGES`
in `ui/litellm-dashboard/src/utils/migratedPages.ts`). Both suites pick it up
automatically.
Add an entry to `tests/e2e/ui/fixtures/migratedPages.ts`, keyed by the legacy page
ID. Specify its route segment, accessible link name, sidebar group if collapsed,
and distinctive visible content such as a heading or tab. Keep expectations
independent of the application's route table so an incorrect destination fails
the test. Both navigation suites use this fixture
For a licensed-only page, `unlicensedText` describes the expected upgrade notice.
The authenticated session's license claim determines which content must render
## Running
@ -31,4 +39,9 @@ SERVER_ROOT_PATH=/litellm npm run e2e:migration:root
```
`globalSetup` logs in once per role; the admin storage state is reused for these
tests. Under a non-root mount it logs in at `${SERVER_ROOT_PATH}/ui/login`.
tests. Under a non-root mount it logs in at `${SERVER_ROOT_PATH}/ui/login`
`tests/navigation/sidebar.spec.ts` also checks the navigation helpers against
equivalent link formats on the live dashboard and a deep link containing a query
string and fragment. The link-format cases change only the rendered `href`
attribute to exercise the locator contract; destination pages and APIs remain live

View file

@ -1,105 +1,73 @@
import { test, expect, type Page } from "@playwright/test";
import { MIGRATED_E2E_SEGMENTS } from "../../fixtures/migratedPages";
import { MIGRATED_E2E_PAGES, type MigratedPage } from "../../fixtures/migratedPages";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { dismissFeedbackPopup } from "../../helpers/navigation";
import { clickSidebarLink, dismissFeedbackPopup, expectUiRoute, sidebarLink } from "../../helpers/navigation";
import { proxyIsPremium } from "../../helpers/premium";
/**
* App Router migration smoke as a user journey: start where the proxy lands you,
* click a migrated page in the sidebar, confirm it routed and rendered, reload it
* (the check a wrong server_root_path breaks), bounce to a legacy page and back,
* and, once two pages are migrated, navigate directly between two migrated pages.
*
* Driven by MIGRATED_E2E_SEGMENTS, so it grows as pages are migrated. Set
* SERVER_ROOT_PATH (e.g. "/litellm") to exercise the non-root mount; leave it
* unset for the default mount. Boot the proxy with the matching value first.
*/
const ROOT = process.env.SERVER_ROOT_PATH ?? "";
const ROOT = (process.env.SERVER_ROOT_PATH ?? "").replace(/\/+$/, "");
const apiKeys = MIGRATED_E2E_PAGES["api-keys"];
const esc = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const pathRe = (segment: string) => new RegExp(`${esc(ROOT)}/ui/${esc(segment)}/?($|\\?)`);
// Scope nav lookups to the sidebar (a `complementary` landmark). The top bar
// now renders a breadcrumb whose current-page item is also a "Virtual Keys"
// link, so an unscoped locator would match two elements.
const sidebar = (page: Page) => page.getByRole("complementary");
const virtualKeysLink = (page: Page) => sidebar(page).getByRole("link", { name: "Virtual Keys", exact: true });
/** The dashboard shell is present (sidebar rendered); page didn't 404 / crash. */
async function expectRendered(page: Page) {
await expect(virtualKeysLink(page)).toBeVisible({ timeout: 20_000 });
async function expectContent(page: Page, destination: MigratedPage): Promise<void> {
await expect(sidebarLink(page, apiKeys.linkName)).toBeVisible({ timeout: 20_000 });
const main = page.getByRole("main");
if (destination.unlicensedText && !proxyIsPremium()) {
await expect(main.getByText(destination.unlicensedText, { exact: true })).toBeVisible();
return;
}
const content = destination.content;
const landmark =
"role" in content
? main.getByRole(content.role, { name: content.name, exact: true })
: main.getByText(content.text, { exact: true });
await expect(landmark).toBeVisible();
}
/**
* Click a migrated page's sidebar link. Migrated items render as <a href=".../ui/<segment>">;
* nested ones live under collapsible groups whose children only render while the
* group is open, so expand collapsed groups until the link is clickable.
*/
async function clickSidebar(page: Page, segment: string) {
const link = sidebar(page).locator(`a[href$="/ui/${segment}"]`).first();
const collapsedGroups = sidebar(page).getByRole("button", { expanded: false });
for (let i = 0; i < 8 && !(await link.isVisible().catch(() => false)); i++) {
const stillCollapsed = await collapsedGroups.count();
if (stillCollapsed === 0) break;
await collapsedGroups.first().click();
await expect(collapsedGroups).toHaveCount(stillCollapsed - 1);
}
await link.click();
async function navigateToDestination(page: Page, destination: MigratedPage): Promise<void> {
await clickSidebarLink(page, destination.linkName, destination.group);
await expectUiRoute(page, destination.segment);
await dismissFeedbackPopup(page);
await expectContent(page, destination);
}
test.use({ storageState: ADMIN_STORAGE_PATH });
test.describe("App Router migrated pages", () => {
for (const segment of MIGRATED_E2E_SEGMENTS) {
test(`${segment}: sidebar nav, reload, and round-trip via the api-keys landing`, async ({ page }) => {
for (const destination of Object.values(MIGRATED_E2E_PAGES)) {
test(`${destination.segment}: sidebar nav, reload, and round-trip via the api-keys landing`, async ({ page }) => {
const pageErrors: string[] = [];
page.on("pageerror", (e) => pageErrors.push(String(e)));
page.on("pageerror", (error) => pageErrors.push(String(error)));
// 1. Start where the proxy lands us.
await page.goto(`${ROOT}/ui/`);
const landing = await page.goto(`${ROOT}/ui/`);
expect(landing?.ok(), "dashboard document loads successfully").toBe(true);
await dismissFeedbackPopup(page);
await expectRendered(page);
await expectContent(page, apiKeys);
// 2. Click the migrated page in the sidebar -> path route + rendered.
await clickSidebar(page, segment);
await expect(page).toHaveURL(pathRe(segment));
await expectRendered(page);
// 3. Reload the path route directly; a wrong server_root_path 404s here.
await page.reload();
await navigateToDestination(page, destination);
const reloaded = await page.reload();
expect(reloaded?.ok(), `${destination.segment} document loads on reload`).toBe(true);
await expectUiRoute(page, destination.segment);
await dismissFeedbackPopup(page);
await expect(page).toHaveURL(pathRe(segment));
await expectRendered(page);
// 4. Click the Virtual Keys sidebar link to the api-keys landing (now a path route), then back.
await virtualKeysLink(page).click();
await expect(page).toHaveURL(pathRe("api-keys"));
await dismissFeedbackPopup(page);
await expectRendered(page);
// 5. Click back to the migrated page.
await clickSidebar(page, segment);
await expect(page).toHaveURL(pathRe(segment));
await expectRendered(page);
expect(pageErrors, `page errors during ${segment} journey`).toEqual([]);
await expectContent(page, destination);
await navigateToDestination(page, apiKeys);
await navigateToDestination(page, destination);
expect(pageErrors, `page errors during ${destination.segment} journey`).toEqual([]);
});
}
test("navigates directly between two migrated pages", async ({ page }) => {
test.skip(MIGRATED_E2E_SEGMENTS.length < 2, "needs >= 2 migrated pages");
const [first, second] = MIGRATED_E2E_SEGMENTS;
const pageErrors: string[] = [];
page.on("pageerror", (e) => pageErrors.push(String(e)));
page.on("pageerror", (error) => pageErrors.push(String(error)));
await page.goto(`${ROOT}/ui/`);
const landing = await page.goto(`${ROOT}/ui/`);
expect(landing?.ok(), "dashboard document loads successfully").toBe(true);
await dismissFeedbackPopup(page);
await expectContent(page, apiKeys);
await clickSidebar(page, first);
await expect(page).toHaveURL(pathRe(first));
await expectRendered(page);
await clickSidebar(page, second);
await expect(page).toHaveURL(pathRe(second));
await expectRendered(page);
// Back to the first migrated page.
await clickSidebar(page, first);
await expect(page).toHaveURL(pathRe(first));
await expectRendered(page);
expect(pageErrors, "page errors during migrated -> migrated nav").toEqual([]);
for (const destination of [apiKeys, MIGRATED_E2E_PAGES.models, apiKeys]) {
await navigateToDestination(page, destination);
}
expect(pageErrors, "page errors during migrated page navigation").toEqual([]);
});
});

View file

@ -3,7 +3,13 @@ import { Role } from "../../fixtures/roles";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { Page } from "../../fixtures/pages";
import { menuLabelToPage } from "../../fixtures/menuMappings";
import { navigateToPage } from "../../helpers/navigation";
import {
clickSidebarLink,
dismissFeedbackPopup,
expectUiRoute,
navigateToPage,
sidebarLink,
} from "../../helpers/navigation";
import { MIGRATED_E2E_PAGES } from "../../fixtures/migratedPages";
import type { Page as PlaywrightPage } from "@playwright/test";
@ -11,7 +17,7 @@ const sidebarButtons = {
[Role.ProxyAdmin]: [
"Virtual Keys",
"Playground",
"Models",
"Models + Endpoints",
"Usage",
"Teams",
"Internal Users",
@ -22,9 +28,9 @@ const sidebarButtons = {
/** Migrated pages live at a path route; legacy pages keep the ?page= query param. */
async function expectPageUrl(page: PlaywrightPage, pageKey: string): Promise<void> {
const migratedSegment = MIGRATED_E2E_PAGES[pageKey];
if (migratedSegment) {
await expect(page).toHaveURL(new RegExp(`/ui/${migratedSegment}/?($|\\?)`));
const migratedPage = MIGRATED_E2E_PAGES[pageKey];
if (migratedPage) {
await expectUiRoute(page, migratedPage.segment);
} else {
await expect(page).toHaveURL(new RegExp(`[?&]page=${pageKey}(&|$)`));
}
@ -51,12 +57,7 @@ for (const { role, storage } of roles) {
throw new Error(`No page mapping found for menu label: ${buttonLabel}`);
}
// Sidebar items are links inside the `complementary` landmark; scoping
// there avoids the top-bar breadcrumb, which also links the page name.
const tab = page.getByRole("complementary").getByRole("link", { name: buttonLabel });
await expect(tab).toBeVisible();
await tab.click();
await clickSidebarLink(page, buttonLabel);
await expectPageUrl(page, expectedPage);
}
@ -81,5 +82,41 @@ for (const { role, storage } of roles) {
await navigateToPage(page, Page.LlmPlayground);
await expectPageUrl(page, Page.LlmPlayground);
});
for (const format of ["without trailing slash", "absolute with query and fragment", "relative"] as const) {
test(`sidebar locator tolerates hrefs ${format}`, async ({ page }) => {
await page.goto("/ui/");
await dismissFeedbackPopup(page);
const link = sidebarLink(page, "Models + Endpoints");
await expect(link).toBeVisible();
const destination = new URL("/ui/models-and-endpoints/", page.url());
const href =
format === "without trailing slash"
? destination.pathname.replace(/\/$/, "")
: format === "relative"
? "./models-and-endpoints/"
: `${destination.href}?source=navigation-smoke#overview`;
await link.evaluate((element, value) => element.setAttribute("href", value), href);
await expect(link).toHaveAttribute("href", href);
await clickSidebarLink(page, "Models + Endpoints");
await expectUiRoute(page, "models-and-endpoints");
await expect(
page.getByRole("main").getByRole("heading", { name: "Model Management", exact: true }),
).toBeVisible();
});
}
test("route assertion tolerates a query string and fragment on a deep link", async ({ page }) => {
const response = await page.goto("/ui/models-and-endpoints/?source=navigation-smoke#overview");
expect(response?.ok()).toBe(true);
await expectUiRoute(page, "models-and-endpoints");
await expect(
page.getByRole("main").getByRole("heading", { name: "Model Management", exact: true }),
).toBeVisible();
expect(new URL(page.url()).search).toBe("?source=navigation-smoke");
expect(new URL(page.url()).hash).toBe("#overview");
});
});
}