test(e2e): move Admin UI Playwright suite to tests/e2e/ui (#34196)

Relocates ui/litellm-dashboard/e2e_tests to tests/e2e/ui so all end to end
suites live under tests/e2e. The suite stays in TypeScript and becomes a
self-contained npm package with its own package.json, lockfile and tsconfig
instead of leaning on the dashboard's toolchain; the dashboard drops its
@playwright/test dependency, e2e scripts and knip/vitest/tsconfig carve-outs.

CI paths follow the move: both CircleCI jobs (main e2e and the
SERVER_ROOT_PATH migration smoke) and the test_server_root_path workflow now
install and run Playwright from tests/e2e/ui, with the node cache keyed on
both lockfiles. classify_changes.sh treats tests/e2e/ui as client so spec
edits keep skipping backend jobs. The suite's mock LLM fixture is excluded
from the e2e basedpyright zero-error gate in pyrightconfig.json since it
belongs to the TS suite, not the typed Python harness.
This commit is contained in:
ryan-crabbe-berri 2026-07-22 12:43:10 -07:00 committed by GitHub
parent 17a83aa896
commit 0fcaadf11c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 194 additions and 55 deletions

View file

@ -2731,7 +2731,7 @@ jobs:
- ~/.cache/uv
- restore_cache:
keys:
- ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
# The cimg/python:3.12-browsers image already ships the Chromium system
@ -2742,11 +2742,14 @@ jobs:
command: |
cd ui/litellm-dashboard
npm ci
cd ../../tests/e2e/ui
npm ci
npx playwright install chromium
- save_cache:
key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- tests/e2e/ui/node_modules
- ~/.cache/ms-playwright
- run:
name: Build UI from source
@ -2777,10 +2780,10 @@ jobs:
name: Seed database
command: |
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
-f tests/e2e/ui/fixtures/seed.sql
- run:
name: Start mock LLM server
command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
background: true
- run:
name: Start LiteLLM proxy
@ -2798,7 +2801,7 @@ jobs:
command: |
LITELLM_LICENSE="$LITELLM_LICENSE" \
uv run --no-sync python -m litellm.proxy.proxy_cli \
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
--config tests/e2e/ui/fixtures/config.yml \
--port 4000
background: true
- run:
@ -2819,15 +2822,15 @@ jobs:
# Forward LITELLM_LICENSE so license.spec.ts can detect that the
# proxy was launched with a license and assert premium_user=true.
command: |
cd ui/litellm-dashboard
cd tests/e2e/ui
LITELLM_LICENSE="$LITELLM_LICENSE" \
npx playwright test --config e2e_tests/playwright.config.ts
npx playwright test --config playwright.config.ts
no_output_timeout: 10m
- store_artifacts:
path: ui/litellm-dashboard/test-results
path: tests/e2e/ui/test-results
destination: e2e-test-results
- store_artifacts:
path: ui/litellm-dashboard/playwright-report
path: tests/e2e/ui/playwright-report
destination: e2e-playwright-report
e2e_ui_testing_server_root_path:
@ -2870,17 +2873,20 @@ jobs:
- ~/.cache/uv
- restore_cache:
keys:
- ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
- run:
name: Install Node dependencies and Playwright
command: |
cd ui/litellm-dashboard
npm ci
cd ../../tests/e2e/ui
npm ci
npx playwright install chromium
- save_cache:
key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
key: ui-e2e-node-deps-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- tests/e2e/ui/node_modules
- ~/.cache/ms-playwright
- run:
name: Build UI from source
@ -2902,10 +2908,10 @@ jobs:
name: Seed database
command: |
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
-f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
-f tests/e2e/ui/fixtures/seed.sql
- run:
name: Start mock LLM server
command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
background: true
- run:
name: Start LiteLLM proxy under a server root path
@ -2918,7 +2924,7 @@ jobs:
command: |
LITELLM_LICENSE="$LITELLM_LICENSE" \
uv run --no-sync python -m litellm.proxy.proxy_cli \
--config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
--config tests/e2e/ui/fixtures/config.yml \
--port 4000
background: true
- run:
@ -2937,15 +2943,15 @@ jobs:
- run:
name: Run migration smoke under SERVER_ROOT_PATH
command: |
cd ui/litellm-dashboard
cd tests/e2e/ui
LITELLM_LICENSE="$LITELLM_LICENSE" \
npx playwright test --config e2e_tests/migration.serverRootPath.config.ts
npx playwright test --config migration.serverRootPath.config.ts
no_output_timeout: 10m
- store_artifacts:
path: ui/litellm-dashboard/test-results
path: tests/e2e/ui/test-results
destination: e2e-server-root-path-test-results
- store_artifacts:
path: ui/litellm-dashboard/playwright-report
path: tests/e2e/ui/playwright-report
destination: e2e-server-root-path-playwright-report
build_docker_database_image:

View file

@ -8,7 +8,7 @@ has_backend=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
case "$file" in
ui/*) has_client=true ;;
ui/* | tests/e2e/ui/*) has_client=true ;;
docs/* | *.md | *.mdx) : ;;
*) has_backend=true ;;
esac

View file

@ -106,8 +106,8 @@ jobs:
with:
node-version: "20"
- name: Install UI deps and Chromium
working-directory: ui/litellm-dashboard
- name: Install e2e deps and Chromium
working-directory: tests/e2e/ui
run: |
retry() {
local attempt=1
@ -131,17 +131,17 @@ jobs:
retry npx playwright install --with-deps chromium
- name: Run SERVER_ROOT_PATH redirect e2e
working-directory: ui/litellm-dashboard
working-directory: tests/e2e/ui
env:
SERVER_ROOT_PATH: ${{ matrix.root_path }}
run: npx playwright test --config=e2e_tests/serverRootPath.config.ts
run: npx playwright test --config=serverRootPath.config.ts
- name: Upload Playwright artifacts on failure
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: playwright-trace-${{ strategy.job-index }}
path: ui/litellm-dashboard/test-results/
path: tests/e2e/ui/test-results/
retention-days: 7
- name: Cleanup

View file

@ -1,7 +1,7 @@
{
"include": ["litellm"],
"ignore": [],
"exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "litellm/types/utils.py", "litellm/proxy/_types.py"],
"exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "tests/e2e/ui", "litellm/types/utils.py", "litellm/proxy/_types.py"],
"pythonVersion": "3.12",
"typeCheckingMode": "strict",
"enableTypeIgnoreComments": false,

View file

@ -22,6 +22,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`
## MCP suite: real Datadog only

View file

@ -62,7 +62,7 @@ class TestSummarizePlannedTurns:
class TestRetried:
def test_transient_failures_then_success_returns_the_success(self) -> None:
outcome = Success(data=SessionMessagesResponse())
outcome = Success[SessionMessagesResponse](status_code=200, data=SessionMessagesResponse())
calls = iter(
(NetworkError(message="overloaded"), NetworkError(message="overloaded"), outcome)
)
@ -88,7 +88,7 @@ class TestRetried:
raise AssertionError("slept after a successful attempt")
result = retried(
lambda: Success(data=SessionMessagesResponse()),
lambda: Success[SessionMessagesResponse](status_code=200, data=SessionMessagesResponse()),
attempts=3,
sleep=sleep_means_retry,
)

111
tests/e2e/ui/package-lock.json generated Normal file
View file

@ -0,0 +1,111 @@
{
"name": "litellm-ui-e2e",
"version": "0.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "litellm-ui-e2e",
"version": "0.0.0",
"devDependencies": {
"@playwright/test": "1.58.1",
"@types/node": "20.19.37",
"typescript": "5.9.3"
}
},
"node_modules/@playwright/test": {
"version": "1.58.1",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz",
"integrity": "sha512-6LdVIUERWxQMmUSSQi0I53GgCBYgM2RpGngCPY7hSeju+VrKjq3lvs7HpJoPbDiY5QM5EYRtRX5fvrinnMAz3w==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright": "1.58.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=18"
}
},
"node_modules/@types/node": {
"version": "20.19.37",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.37.tgz",
"integrity": "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.21.0"
}
},
"node_modules/fsevents": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/playwright": {
"version": "1.58.1",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz",
"integrity": "sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==",
"dev": true,
"license": "Apache-2.0",
"dependencies": {
"playwright-core": "1.58.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=18"
},
"optionalDependencies": {
"fsevents": "2.3.2"
}
},
"node_modules/playwright-core": {
"version": "1.58.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.1.tgz",
"integrity": "sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"playwright-core": "cli.js"
},
"engines": {
"node": ">=18"
}
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "6.21.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
"dev": true,
"license": "MIT"
}
}
}

16
tests/e2e/ui/package.json Normal file
View file

@ -0,0 +1,16 @@
{
"name": "litellm-ui-e2e",
"version": "0.0.0",
"private": true,
"scripts": {
"e2e": "playwright test --config playwright.config.ts",
"e2e:ui": "playwright test --ui --config playwright.config.ts",
"e2e:migration": "playwright test tests/migration/migratedPages.spec.ts --config playwright.config.ts",
"e2e:migration:root": "playwright test --config migration.serverRootPath.config.ts"
},
"devDependencies": {
"@playwright/test": "1.58.1",
"@types/node": "20.19.37",
"typescript": "5.9.3"
}
}

View file

@ -20,8 +20,8 @@ set -euo pipefail
# ================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DASHBOARD_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
DASHBOARD_DIR="$REPO_ROOT/ui/litellm-dashboard"
IS_CI="${CI:-false}"
CONTAINER_NAME="litellm-e2e-postgres-$$"
MOCK_PID=""
@ -187,12 +187,12 @@ PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAM
# --- Playwright ---
echo "=== Installing Playwright dependencies ==="
cd "$DASHBOARD_DIR"
cd "$SCRIPT_DIR"
npm install --silent 2>/dev/null || true
npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium
echo "=== Running Playwright tests ==="
npx playwright test --config e2e_tests/playwright.config.ts "$@"
npx playwright test --config playwright.config.ts "$@"
EXIT_CODE=$?
exit $EXIT_CODE

View file

@ -9,8 +9,9 @@ the default mount and a non-root `SERVER_ROOT_PATH` mount.
## Adding a page
When a page's migration merges, add its route segment to
`e2e_tests/fixtures/migratedPages.ts` (keep it in lockstep with `MIGRATED_PAGES`
in `src/utils/migratedPages.ts`). Both suites pick it up automatically.
`tests/e2e/ui/fixtures/migratedPages.ts` (keep it in lockstep with `MIGRATED_PAGES`
in `ui/litellm-dashboard/src/utils/migratedPages.ts`). Both suites pick it up
automatically.
## Running

View file

@ -6,7 +6,7 @@ import { Role, users } from "../../fixtures/users";
// Type-only import of the OpenAPI-generated backend schema, erased at runtime by
// esbuild. It types the round-trips below so mistakes surface in the editor; the live
// test against the real proxy is what actually enforces the contract.
import type { components } from "../../../src/lib/http/schema";
import type { components } from "../../../../../ui/litellm-dashboard/src/lib/http/schema";
// These tests mutate the proxy's shared router_settings, and the Loadbalancing save
// echoes the whole settings object, so they must not run concurrently.

View file

@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "commonjs",
"moduleResolution": "node",
"lib": ["ES2022", "DOM"],
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"resolveJsonModule": true,
"noEmit": true,
"types": ["node"]
},
"include": ["**/*.ts"],
"exclude": ["node_modules"]
}

View file

@ -7756,6 +7756,7 @@ async def test_cli_completion_persists_assertion_under_db_user_id():
user_defined_values=None,
prisma_client=MagicMock(),
user_api_key_cache=MagicMock(),
cli_sso_session_cache=MagicMock(),
proxy_logging_obj=MagicMock(),
sso_assertion=assertion,
)

View file

@ -1,7 +1,7 @@
{
"$schema": "https://unpkg.com/knip@5/schema.json",
"entry": ["scripts/**/*.{ts,mjs}", "src/components/ui/**/*.{ts,tsx}"],
"project": ["src/**/*.{ts,tsx}", "tests/**/*.{ts,tsx}", "scripts/**/*.{ts,mjs}", "e2e_tests/**/*.ts"],
"project": ["src/**/*.{ts,tsx}", "tests/**/*.{ts,tsx}", "scripts/**/*.{ts,mjs}"],
"ignore": ["src/lib/http/schema.d.ts"],
"ignoreDependencies": [
"openapi-typescript",
@ -10,14 +10,6 @@
"tailwindcss",
"tw-animate-css"
],
"playwright": {
"config": [
"e2e_tests/playwright.config.ts",
"e2e_tests/serverRootPath.config.ts",
"e2e_tests/migration.serverRootPath.config.ts"
],
"entry": ["e2e_tests/**/*.spec.ts", "e2e_tests/**/*.setup.ts", "e2e_tests/globalSetup.ts"]
},
"vitest": {
"config": ["vitest.config.ts"]
},

View file

@ -47,7 +47,6 @@
},
"devDependencies": {
"@eslint/js": "9.39.2",
"@playwright/test": "1.58.1",
"@tailwindcss/forms": "0.5.11",
"@tailwindcss/postcss": "4.3.2",
"@testing-library/dom": "10.4.1",
@ -2723,8 +2722,9 @@
"version": "1.58.1",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz",
"integrity": "sha512-6LdVIUERWxQMmUSSQi0I53GgCBYgM2RpGngCPY7hSeju+VrKjq3lvs7HpJoPbDiY5QM5EYRtRX5fvrinnMAz3w==",
"devOptional": true,
"license": "Apache-2.0",
"optional": true,
"peer": true,
"dependencies": {
"playwright": "1.58.1"
},
@ -7361,7 +7361,6 @@
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"optional": true,
@ -10948,8 +10947,9 @@
"version": "1.58.1",
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz",
"integrity": "sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==",
"devOptional": true,
"license": "Apache-2.0",
"optional": true,
"peer": true,
"dependencies": {
"playwright-core": "1.58.1"
},
@ -10967,8 +10967,9 @@
"version": "1.58.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.1.tgz",
"integrity": "sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==",
"devOptional": true,
"license": "Apache-2.0",
"optional": true,
"peer": true,
"bin": {
"playwright-core": "cli.js"
},

View file

@ -14,10 +14,6 @@
"test:coverage": "vitest run --coverage",
"format": "prettier --write .",
"format:check": "prettier --check .",
"e2e": "playwright test --config e2e_tests/playwright.config.ts",
"e2e:ui": "playwright test --ui --config e2e_tests/playwright.config.ts",
"e2e:migration": "playwright test e2e_tests/tests/migration/migratedPages.spec.ts --config e2e_tests/playwright.config.ts",
"e2e:migration:root": "playwright test --config e2e_tests/migration.serverRootPath.config.ts",
"knip": "knip",
"knip:ci": "knip --exclude exports,nsExports,types,nsTypes,enumMembers,classMembers,duplicates",
"knip:fix": "knip --fix",
@ -63,7 +59,6 @@
},
"devDependencies": {
"@eslint/js": "9.39.2",
"@playwright/test": "1.58.1",
"@tailwindcss/forms": "0.5.11",
"@tailwindcss/postcss": "4.3.2",
"@testing-library/dom": "10.4.1",

View file

@ -23,5 +23,5 @@
"target": "ES2017"
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", ".next/dev/types/**/*.ts"],
"exclude": ["node_modules", "e2e_tests", "scripts"]
"exclude": ["node_modules", "scripts"]
}

View file

@ -32,7 +32,6 @@ const config: ViteUserConfig = {
"**/*.spec.*",
"tests/**",
"e2e_tests/**",
"node_modules/**",
".next/**",
@ -44,7 +43,7 @@ const config: ViteUserConfig = {
"next.config.*",
],
},
exclude: ["e2e_tests/**", "node_modules/**"],
exclude: ["node_modules/**"],
include: ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.test.ts", "tests/**/*.test.tsx"],
},
resolve: {