diff --git a/ui/litellm-dashboard/e2e_tests/parity/README.md b/ui/litellm-dashboard/e2e_tests/parity/README.md new file mode 100644 index 00000000000..622b50ea079 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/parity/README.md @@ -0,0 +1,63 @@ +# Phase-1 parity specs + +One Playwright spec file per migrated section: `
.spec.ts`. + +## Conventions + +- **Assert on user-observable behavior only:** `getByRole`, `getByText`, + `getByLabel`, `getByTestId`. Never assert on DOM selectors that identify the + framework (e.g. `.ant-btn`). +- **Visual snapshots** live in `
.spec.ts-snapshots/` next to each + spec. `expect(page).toHaveScreenshot()` is the canonical call. +- **Both old and new UI pass:** specs must pass against the current UI + (sanity) before the section is migrated, and against the migrated UI after. +- **Exception:** when a `QUIRKS.md` entry uses resolution `fix`, the spec + captures the corrected behavior; the old-UI sanity check is explicitly + skipped (record the skip in `QUIRKS.md`). +- **Soft cap:** ~30 assertions per spec. If the section genuinely needs more, + split into `
-a.spec.ts` / `
-b.spec.ts`. + +## Running + +Default project target is the local Next dev server at +`http://localhost:3000` (started automatically by `webServer` in +`playwright.config.ts`): + +```bash +npx playwright test --project=parity e2e_tests/parity/
.spec.ts +``` + +To run parity specs against the proxy instead (e.g. because dev-server auth +mocking is too brittle for the section), set `PARITY_BASE_URL`: + +```bash +PARITY_BASE_URL=http://localhost:4000 \ + npx playwright test --project=parity e2e_tests/parity/
.spec.ts +``` + +When `PARITY_BASE_URL` is set, the dev-server `webServer` hook does not +start; you are responsible for having the target server running. + +## Snapshot baselining + +First run per section: create baselines. + +```bash +npx playwright test --project=parity \ + e2e_tests/parity/
.spec.ts --update-snapshots +``` + +Subsequent runs (verification): no `--update-snapshots`. Pixel diffs count as +regressions. Intentional rebaselines are logged in `docs/DEVIATIONS.md`. + +## Authentication + +Dev-server auth is a known weak spot for phase-1 parity. Options per spec: + +1. **Playwright route interception** — stub the `/get/*` endpoints the UI hits + during boot (`get/ui_theme_settings`, `get/sso_settings`, etc.) + seed + `sessionStorage` with a fake JWT so useAuthorized returns admin. +2. **Use `PARITY_BASE_URL=http://localhost:4000`** — run against the real + proxy with a seeded admin user. Slower but more realistic. + +Pick per section depending on how much the section exercises backend calls. diff --git a/ui/litellm-dashboard/e2e_tests/playwright.config.ts b/ui/litellm-dashboard/e2e_tests/playwright.config.ts index ec4d3a6ddb0..a2953072edf 100644 --- a/ui/litellm-dashboard/e2e_tests/playwright.config.ts +++ b/ui/litellm-dashboard/e2e_tests/playwright.config.ts @@ -2,7 +2,19 @@ import { defineConfig, devices } from "@playwright/test"; /** * See https://playwright.dev/docs/test-configuration. + * + * Two projects: + * - `chromium`: existing full e2e suite against the proxy at :4000. + * - `parity`: phase-1 shadcn migration parity specs under `./parity/`. + * Uses a dev-server baseURL (3000) by default but can be + * overridden via `PARITY_BASE_URL` (e.g. to aim at the + * proxy at 4000 if dev-server auth mocking proves too + * brittle). The webServer entry starts `npm run dev` + * automatically if PARITY_BASE_URL is not set. */ +const PARITY_BASE_URL = process.env.PARITY_BASE_URL ?? "http://localhost:3000"; +const PARITY_USES_LOCAL_DEV = !process.env.PARITY_BASE_URL; + export default defineConfig({ testDir: ".", testMatch: ["**/*.spec.ts", "**/*.setup.ts"], @@ -34,14 +46,45 @@ export default defineConfig({ projects: [ { name: "chromium", + testDir: ".", + testIgnore: ["parity/**", "**/*.test.*"], use: { ...devices["Desktop Chrome"] }, }, + { + name: "parity", + testDir: "./parity", + testMatch: ["**/*.spec.ts"], + use: { + ...devices["Desktop Chrome"], + baseURL: PARITY_BASE_URL, + // Lock viewport for stable visual snapshots across environments. + viewport: { width: 1440, height: 900 }, + }, + }, ], + /* Auto-start the Next dev server for parity specs unless PARITY_BASE_URL + points elsewhere (e.g. the proxy at :4000 for auth-backed parity runs). */ + webServer: PARITY_USES_LOCAL_DEV + ? { + command: "npm run dev", + cwd: "..", + url: "http://localhost:3000", + reuseExistingServer: true, + timeout: 120 * 1000, + } + : undefined, + /* Timeout settings */ timeout: 3 * 60 * 1000, expect: { timeout: 10 * 1000, + /* Keep visual snapshot diffs tight so real regressions aren't masked. */ + toHaveScreenshot: { + maxDiffPixelRatio: 0.01, + animations: "disabled", + caret: "hide", + }, }, globalSetup: require.resolve("./globalSetup"), });