From 0f82405621b9a8c7d12bf7faedfd729f38fb4b34 Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Wed, 8 Jul 2026 20:26:32 -0400 Subject: [PATCH] test(e2e): add Playwright UI suite for admin virtual-key creation Adds a standalone Playwright (TypeScript) suite under tests/e2e/ui that drives the Admin UI in a real browser. The single spec logs in as the proxy admin, creates a virtual key scoped to one model through the create-key modal, then asserts both halves of the contract: /key/info reflects the alias and model the form set, and the generated key serves /chat/completions on its scoped model while being denied key_model_access_denied on a model outside that scope. The suite is decoupled from the Python harness in the sibling folders; it carries its own package.json and playwright.config.ts and runs with npm test. --- tests/e2e/CLAUDE.md | 1 + tests/e2e/ui/.gitignore | 6 ++ tests/e2e/ui/README.md | 40 +++++++++ tests/e2e/ui/create-virtual-key.spec.ts | 75 ++++++++++++++++ tests/e2e/ui/package-lock.json | 111 ++++++++++++++++++++++++ tests/e2e/ui/package.json | 16 ++++ tests/e2e/ui/playwright.config.ts | 29 +++++++ tests/e2e/ui/tsconfig.json | 14 +++ 8 files changed, 292 insertions(+) create mode 100644 tests/e2e/ui/.gitignore create mode 100644 tests/e2e/ui/README.md create mode 100644 tests/e2e/ui/create-virtual-key.spec.ts create mode 100644 tests/e2e/ui/package-lock.json create mode 100644 tests/e2e/ui/package.json create mode 100644 tests/e2e/ui/playwright.config.ts create mode 100644 tests/e2e/ui/tsconfig.json diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 20ab0212854..a07aeebe85a 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -15,6 +15,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `spend_tracking/` - spend logging and cost attribution on `/spend/*` - `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials - `logging/` - logging-integration delivery (datadog and friends) +- `ui/` - Admin UI browser flows, a standalone Playwright (TypeScript) project separate from the Python harness above: an admin walks a real page (login, create/edit resources) and the test asserts both the UI outcome and, via the proxy API, that the action persisted and the gateway enforces it. Its own `package.json`/`playwright.config.ts`; run with `npm test` from that folder, not pytest - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (rate limits, fallbacks, cooldowns) - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests diff --git a/tests/e2e/ui/.gitignore b/tests/e2e/ui/.gitignore new file mode 100644 index 00000000000..6b0908574cd --- /dev/null +++ b/tests/e2e/ui/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +test-results/ +playwright-report/ +blob-report/ +playwright/.cache/ +*.storageState.json diff --git a/tests/e2e/ui/README.md b/tests/e2e/ui/README.md new file mode 100644 index 00000000000..99366e0c5a4 --- /dev/null +++ b/tests/e2e/ui/README.md @@ -0,0 +1,40 @@ +# Admin UI e2e (Playwright) + +A standalone Playwright suite that drives the LiteLLM Admin UI in a real browser, separate from the Python harness in the sibling folders. It proves the flows a proxy admin actually clicks through, and cross-checks each one against the proxy API so a green test means the browser action really took effect on the server + +The single spec here logs in as the proxy admin, creates a virtual key scoped to one model through the create-key modal, and then asserts both halves of the contract: `/key/info` reflects the alias and model the form set (the write persisted), and the generated key serves `/chat/completions` on its scoped model while being denied `key_model_access_denied` on a model outside that scope (the gateway enforces the scope) + +## Prerequisites + +You need a running proxy that serves the UI on `http://localhost:4000` (the repo's `tests/e2e/docker-compose.yml` brings one up with the `gemini-2.5-flash` and `gpt-5.5` models this spec uses). The admin credentials are username `admin` and password = the proxy master key + +## Running + +1. Install dependencies and the browser from this folder: + + ```bash + npm install + npm run install:browser + ``` + +2. Bring up a proxy (from `tests/e2e/`): + + ```bash + docker compose up -d + curl -fs http://localhost:4000/health/liveliness + ``` + +3. Run the suite: + + ```bash + npm test + ``` + +## Configuration + +The base URL and credentials come from the environment so the same spec runs against localhost or a deployed proxy + +- `LITELLM_PROXY_URL` (default `http://localhost:4000`) +- `LITELLM_MASTER_KEY` (default `sk-1234`) +- `E2E_UI_ALLOWED_MODEL` (default `gemini-2.5-flash`), the model the key is scoped to +- `E2E_UI_DENIED_MODEL` (default `gpt-5.5`), a model outside the scope used to assert the denial diff --git a/tests/e2e/ui/create-virtual-key.spec.ts b/tests/e2e/ui/create-virtual-key.spec.ts new file mode 100644 index 00000000000..990a4f90445 --- /dev/null +++ b/tests/e2e/ui/create-virtual-key.spec.ts @@ -0,0 +1,75 @@ +import { test, expect, Page } from "@playwright/test"; + +const ALLOWED_MODEL = process.env.E2E_UI_ALLOWED_MODEL ?? "gemini-2.5-flash"; +const DENIED_MODEL = process.env.E2E_UI_DENIED_MODEL ?? "gpt-5.5"; + +const adminAuth = { Authorization: `Bearer sk-1234` }; + +async function loginAsProxyAdmin(page: Page): Promise { + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill("admin"); + await page.getByPlaceholder("Enter your password").fill("sk-1234"); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await expect(page.getByText("Virtual Keys")).toBeVisible({ timeout: 30_000 }); +} + +async function dismissFeedbackPopup(page: Page): Promise { + const dismiss = page.getByText("Don't ask me again"); + if (await dismiss.isVisible({ timeout: 1_500 }).catch(() => false)) { + await dismiss.click(); + } +} + +test.describe("Proxy Admin - create virtual key via the Admin UI", () => { + test("admin creates a model-scoped key and the gateway enforces its scope", async ({ page, request }) => { + await loginAsProxyAdmin(page); + + await page.goto("/ui?page=api-keys"); + await dismissFeedbackPopup(page); + + await page.getByTestId("create-key-button").click(); + await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 15_000 }); + + const alias = `e2e-ui-key-${Date.now()}`; + await page.getByTestId("base-input").fill(alias); + + await page.locator(".ant-select-selection-overflow").click(); + const option = page.locator(".ant-select-dropdown:visible").getByRole("option", { name: ALLOWED_MODEL, exact: true }); + await option.waitFor({ state: "attached" }); + await option.evaluate((el: HTMLElement) => el.click()); + await page.keyboard.press("Escape"); + + await page.getByRole("button", { name: "Create Key", exact: true }).click(); + await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 15_000 }); + + const secret = (await page.locator(".ant-modal:visible pre").innerText()).trim(); + expect(secret).toMatch(/^sk-/); + + await page.keyboard.press("Escape"); + await expect(page.getByText(alias)).toBeVisible({ timeout: 15_000 }); + + try { + const info = await request.get(`/key/info?key=${encodeURIComponent(secret)}`, { headers: adminAuth }); + expect(info.status()).toBe(200); + const infoBody = await info.json(); + expect(infoBody.info.key_alias).toBe(alias); + expect(infoBody.info.models).toEqual([ALLOWED_MODEL]); + + const chat = (model: string) => + request.post("/chat/completions", { + headers: { Authorization: `Bearer ${secret}` }, + data: { model, messages: [{ role: "user", content: "ping" }], max_tokens: 16 }, + }); + + await expect + .poll(async () => (await chat(ALLOWED_MODEL)).status(), { timeout: 60_000, intervals: [2_000] }) + .toBe(200); + + const denied = await chat(DENIED_MODEL); + expect(denied.status()).toBe(403); + expect(await denied.text()).toContain("key_model_access_denied"); + } finally { + await request.post("/key/delete", { headers: adminAuth, data: { keys: [secret] } }); + } + }); +}); diff --git a/tests/e2e/ui/package-lock.json b/tests/e2e/ui/package-lock.json new file mode 100644 index 00000000000..61495c04df7 --- /dev/null +++ b/tests/e2e/ui/package-lock.json @@ -0,0 +1,111 @@ +{ + "name": "litellm-e2e-ui", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "litellm-e2e-ui", + "version": "0.0.0", + "devDependencies": { + "@playwright/test": "1.58.1", + "@types/node": "20.19.37", + "typescript": "5.9.3" + } + }, + "node_modules/@playwright/test": { + "version": "1.58.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.1.tgz", + "integrity": "sha512-6LdVIUERWxQMmUSSQi0I53GgCBYgM2RpGngCPY7hSeju+VrKjq3lvs7HpJoPbDiY5QM5EYRtRX5fvrinnMAz3w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.58.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "20.19.37", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.37.tgz", + "integrity": "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.58.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", + "integrity": "sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.58.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.58.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.1.tgz", + "integrity": "sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/tests/e2e/ui/package.json b/tests/e2e/ui/package.json new file mode 100644 index 00000000000..2592d0f4d83 --- /dev/null +++ b/tests/e2e/ui/package.json @@ -0,0 +1,16 @@ +{ + "name": "litellm-e2e-ui", + "version": "0.0.0", + "private": true, + "description": "Standalone Playwright e2e for the LiteLLM Admin UI", + "scripts": { + "test": "playwright test", + "test:headed": "playwright test --headed", + "install:browser": "playwright install chromium" + }, + "devDependencies": { + "@playwright/test": "1.58.1", + "@types/node": "20.19.37", + "typescript": "5.9.3" + } +} diff --git a/tests/e2e/ui/playwright.config.ts b/tests/e2e/ui/playwright.config.ts new file mode 100644 index 00000000000..661b339b4e8 --- /dev/null +++ b/tests/e2e/ui/playwright.config.ts @@ -0,0 +1,29 @@ +import { defineConfig, devices } from "@playwright/test"; + +const baseURL = process.env.LITELLM_PROXY_URL ?? "http://localhost:4000"; + +export default defineConfig({ + testDir: ".", + testMatch: ["**/*.spec.ts"], + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: "html", + timeout: 3 * 60 * 1000, + expect: { + timeout: 10 * 1000, + }, + use: { + baseURL, + trace: "on-first-retry", + actionTimeout: 15 * 1000, + navigationTimeout: 30 * 1000, + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + ], +}); diff --git a/tests/e2e/ui/tsconfig.json b/tests/e2e/ui/tsconfig.json new file mode 100644 index 00000000000..62658a5057b --- /dev/null +++ b/tests/e2e/ui/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2021", + "module": "CommonJS", + "moduleResolution": "Node", + "lib": ["ES2021", "DOM"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node"], + "noEmit": true + }, + "include": ["**/*.ts"] +}