From 30968fb72132ea9eff18521008f2275e64cd735d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 8 Aug 2026 01:59:10 -0700 Subject: [PATCH] test(e2e-ui): cover the Playground, Logs and Usage manual-QA flows These three pages carried no e2e coverage, so the manual QA checklist was the only thing standing behind them. Playground: sends a chat from the UI for both configured models, and for both virtual-key sources (the logged-in session, and a key pasted into the panel). This is the only spec that drives the dashboard's own LLM call path rather than an admin CRUD endpoint. Logs: a request the proxy actually served appears in the table, its drawer expands to the real request and response bodies, both copy to the clipboard, the Input card collapses, the JSON view exposes Request/Response, and the End User filter narrows the table to one customer. Usage: traffic billed to a virtual key reaches Top Virtual Keys, the card toggles between table and chart, and the key opens its key-info panel. Router settings: the existing spec proved the UI can record a fallback; the new one proves the fallback is honoured, by pointing a model at an unreachable upstream and asserting the reply comes back anyway. It asserts the un-fallen-back call fails first, so a quietly-working primary cannot fake a pass. Supporting changes: - helpers/traffic.ts generates the traffic these pages render, rather than seeding rows no code produced. Its two wait helpers exist because the Logs and Usage pages read different stores: spend logs are flushed on a timer, and the Usage page reads a background rollup *and* fetches once on mount, so waiting on the DOM there can never converge. - helpers/playground.ts holds the playground controls, now shared with the fallback spec. Everything is scoped to the visible copy of the config panel, which is rendered twice for the docked and collapsed layouts. - run_e2e.sh gains E2E_KEEP_ALIVE=1, which brings the stack up and blocks so a spec can be re-run against it without paying for a UI rebuild each iteration. Verified with the full suite on a fresh stack: 89 passed, 0 failed, 5 skipped. --- tests/e2e/ui/helpers/playground.ts | 57 +++++ tests/e2e/ui/helpers/traffic.ts | 153 +++++++++++++ tests/e2e/ui/run_e2e.sh | 73 +++++- tests/e2e/ui/tests/logs/logs.spec.ts | 215 ++++++++++++++++++ .../ui/tests/playground/playground.spec.ts | 56 +++++ .../ui/tests/settings/routerSettings.spec.ts | 118 +++++++++- tests/e2e/ui/tests/usage/usagePage.spec.ts | 90 ++++++++ 7 files changed, 753 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/ui/helpers/playground.ts create mode 100644 tests/e2e/ui/helpers/traffic.ts create mode 100644 tests/e2e/ui/tests/logs/logs.spec.ts create mode 100644 tests/e2e/ui/tests/playground/playground.spec.ts create mode 100644 tests/e2e/ui/tests/usage/usagePage.spec.ts diff --git a/tests/e2e/ui/helpers/playground.ts b/tests/e2e/ui/helpers/playground.ts new file mode 100644 index 00000000000..2664f233748 --- /dev/null +++ b/tests/e2e/ui/helpers/playground.ts @@ -0,0 +1,57 @@ +import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/test"; +import { navigateToPage, dismissFeedbackPopup } from "./navigation"; +import { Page } from "../fixtures/pages"; + +/** + * Controls for the Test Key / Playground page. + * + * Shared because more than one manual-QA flow ends in "send a message from the + * UI and check what comes back" — the playground itself, and router fallbacks, + * which are only really verified by seeing the fallback's reply render. + */ + +/** + * The playground renders its configuration panel twice — once for the docked + * sidebar and once for the collapsed/overlay layout — and only one copy is on + * screen at a time. Every control here is narrowed to the visible copy; an + * unscoped locator hits a strict-mode violation against its hidden twin. + */ +export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first(); + +/** The model dropdown, addressed by the placeholder it shows before selection. */ +export const modelSelect = (page: PlaywrightPage): Locator => + onlyVisible(page.locator('.ant-select:has(.ant-select-selection-placeholder:text-is("Select a Model"))')); + +/** Send button is icon-only (an up-arrow), so there is no accessible name. */ +export const sendButton = (page: PlaywrightPage): Locator => onlyVisible(page.locator("button:has(.anticon-arrow-up)")); + +/** The Virtual Key Source dropdown, addressed by its currently selected label. */ +export const keySourceSelect = (page: PlaywrightPage, current: string): Locator => + onlyVisible(page.locator(`.ant-select:has(.ant-select-selection-item[title="${current}"])`)); + +export async function openPlayground(page: PlaywrightPage): Promise { + await navigateToPage(page, Page.LlmPlayground); + await dismissFeedbackPopup(page); + await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({ + timeout: 20_000, + }); +} + +export async function selectModel(page: PlaywrightPage, model: string): Promise { + const select = modelSelect(page); + await select.click(); + // The dropdown is virtualized: only the options in the rendered window exist + // in the DOM, so once other specs have added models to the proxy the one we + // want is not merely off-screen, it is absent. The Select takes showSearch, + // so type to narrow the list before clicking. + await select.locator("input.ant-select-selection-search-input").fill(model); + // antd portals its dropdown to the body; options carry the value as `title`. + await onlyVisible(page.locator(`.ant-select-item-option[title="${model}"]`)).click({ timeout: 15_000 }); +} + +export async function sendMessage(page: PlaywrightPage, message: string): Promise { + const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false })); + await expect(input).toBeVisible({ timeout: 15_000 }); + await input.fill(message); + await sendButton(page).click(); +} diff --git a/tests/e2e/ui/helpers/traffic.ts b/tests/e2e/ui/helpers/traffic.ts new file mode 100644 index 00000000000..eb958d6376c --- /dev/null +++ b/tests/e2e/ui/helpers/traffic.ts @@ -0,0 +1,153 @@ +import { APIRequestContext, expect } from "@playwright/test"; + +/** + * Helpers for driving real traffic through the proxy from a test. + * + * Several manual-QA flows (Logs, Usage) can only be checked once the proxy has + * actually served a request: the spend log a Logs row renders, and the + * per-key spend the Usage page aggregates, are both written by the completion + * path. Seeding those tables directly would test the UI against rows no code + * produced, so these helpers make the proxy generate them the same way a user + * would. + */ + +/** Model names served by fixtures/config.yml, both backed by the mock LLM server. */ +export const CHAT_MODEL_A = "fake-openai-gpt-4"; +export const CHAT_MODEL_B = "fake-anthropic-claude"; + +/** The only completion text fixtures/mock_llm_server/server.py ever returns. */ +export const MOCK_RESPONSE_TEXT = "This is a mock response."; + +export const masterKey = (): string => process.env.LITELLM_MASTER_KEY || "sk-1234"; + +/** Root path the proxy is mounted under, "" for the default mount. */ +const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? ""; + +interface ChatOptions { + model: string; + prompt: string; + /** Virtual key to bill the call to. Defaults to the master key. */ + apiKey?: string; + /** Sent as `user`, which lands in the spend log's end_user column. */ + endUser?: string; +} + +/** POST /v1/chat/completions and return the completion id (the Logs Request ID). */ +export async function sendChatCompletion(request: APIRequestContext, opts: ChatOptions): Promise { + const res = await request.post(`${rootPath()}/v1/chat/completions`, { + headers: { + Authorization: `Bearer ${opts.apiKey ?? masterKey()}`, + "Content-Type": "application/json", + }, + data: { + model: opts.model, + messages: [{ role: "user", content: opts.prompt }], + ...(opts.endUser ? { user: opts.endUser } : {}), + }, + }); + expect(res.ok(), `chat completion for ${opts.model} failed (${res.status()}): ${await res.text()}`).toBe(true); + const body = await res.json(); + expect(body.choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT); + return body.id as string; +} + +/** + * Create a virtual key via /key/generate. + * + * `key` is the sk- value callers authenticate with; `token` is its hash, which + * is what usage/spend aggregates are keyed by (the plaintext key is never + * stored, so it never appears in those responses). + */ +export async function createVirtualKey( + request: APIRequestContext, + data: Record = {}, +): Promise<{ key: string; token: string; alias?: string }> { + const res = await request.post(`${rootPath()}/key/generate`, { + headers: { + Authorization: `Bearer ${masterKey()}`, + "Content-Type": "application/json", + }, + data, + }); + expect(res.ok(), `key generate failed (${res.status()}): ${await res.text()}`).toBe(true); + const body = await res.json(); + return { + key: body.key as string, + token: (body.token ?? body.token_id) as string, + alias: body.key_alias as string | undefined, + }; +} + +/** + * Spend logs are flushed on a timer, not synchronously with the response, so a + * Logs/Usage assertion made straight after a completion races the writer. Poll + * /spend/logs until the request id shows up rather than sleeping a fixed amount. + */ +export async function waitForSpendLog( + request: APIRequestContext, + requestId: string, + timeoutMs = 60_000, +): Promise { + const deadline = Date.now() + timeoutMs; + let lastStatus = 0; + while (Date.now() < deadline) { + const res = await request.get(`${rootPath()}/spend/logs?request_id=${encodeURIComponent(requestId)}`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + lastStatus = res.status(); + if (res.ok()) { + const body = await res.json(); + const rows = Array.isArray(body) ? body : body?.data ?? []; + if (rows.length > 0) { + return; + } + } + await new Promise((r) => setTimeout(r, 2_000)); + } + throw new Error(`spend log for request ${requestId} never appeared (last /spend/logs status ${lastStatus})`); +} + +const isoDay = (d: Date): string => d.toISOString().slice(0, 10); + +/** + * Wait until a key's traffic has been rolled up into the daily-spend aggregate. + * + * The Usage page is built from /user/daily/activity, which reads the aggregate + * table a background job writes — not the spend log itself. It also fetches + * once on mount and never refetches, so a test that navigates before the + * rollup lands will keep re-reading a stale render until it times out. Waiting + * on the API first is the only way to make that deterministic. + */ +export async function waitForKeyInDailyActivity( + request: APIRequestContext, + keyToken: string, + timeoutMs = 120_000, +): Promise { + const now = new Date(); + const start = new Date(now); + start.setDate(start.getDate() - 7); + const query = `start_date=${isoDay(start)}&end_date=${isoDay(now)}`; + + const deadline = Date.now() + timeoutMs; + let lastStatus = 0; + while (Date.now() < deadline) { + const res = await request.get(`${rootPath()}/user/daily/activity?${query}`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + lastStatus = res.status(); + if (res.ok()) { + const body = await res.json(); + const seen = (body?.results ?? []).some( + (day: { breakdown?: { api_keys?: Record } }) => keyToken in (day.breakdown?.api_keys ?? {}), + ); + if (seen) { + return; + } + } + await new Promise((r) => setTimeout(r, 3_000)); + } + throw new Error( + `key ${keyToken} never appeared in /user/daily/activity (last status ${lastStatus}); ` + + "the daily spend rollup may not be running", + ); +} diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index 858eb401c8e..f85a83a5488 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -33,7 +33,35 @@ if [ "$IS_CI" = "false" ]; then for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do [ -d "$p" ] && export PATH="$p:$PATH" done - [ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh" + # Sourcing nvm only makes `nvm` available -- it leaves you on whatever the + # default alias points at, which is frequently an older Node than the + # dashboard's engines allow. `npm install` then fails EBADENGINE, npm exits + # non-zero, and because the install below is `--silent ... || true` the error + # is swallowed and the run dies later with the far less obvious + # "sh: next: command not found". + # + # So select a Node that satisfies ui/litellm-dashboard's engines.node, and if + # none is available say so here rather than 200 lines downstream. + if [ -s "$HOME/.nvm/nvm.sh" ]; then + # shellcheck disable=SC1091 + source "$HOME/.nvm/nvm.sh" + required_major="$(sed -nE 's/.*"node"[[:space:]]*:[[:space:]]*">=?([0-9]+).*/\1/p' \ + "$DASHBOARD_DIR/package.json" 2>/dev/null | head -1)" + if [ -n "$required_major" ]; then + current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')" + if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then + echo "Node $(node --version 2>/dev/null || echo 'not found') is below the dashboard's required v${required_major}; selecting a newer one via nvm" + nvm use "$required_major" >/dev/null 2>&1 || nvm use --lts >/dev/null 2>&1 || true + current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')" + if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then + echo "Error: ui/litellm-dashboard requires Node >= v${required_major}, and no such version is installed." + echo " Install one with: nvm install ${required_major}" + exit 1 + fi + fi + echo "Using Node $(node --version) / npm $(npm --version)" + fi + fi fi # --- Cleanup on exit --- @@ -59,8 +87,13 @@ if [ "$IS_CI" = "false" ]; then for cmd in docker psql; do command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } done + # Only a LISTENER conflicts with us. Without -sTCP:LISTEN this also matches + # ESTABLISHED sockets, so an unrelated *outbound* connection from this machine + # to someone else's :5432 (a psql session, a running app, a Prisma engine + # talking to a remote database) aborts the run with "port 5432 is in use" + # while nothing is actually bound locally. for port in 4000 5432 8090; do - if lsof -ti ":$port" >/dev/null 2>&1; then + if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then echo "Error: port $port is in use" exit 1 fi @@ -108,7 +141,11 @@ export LITELLM_LICENSE="${LITELLM_LICENSE:-}" # --- Rebuild UI from source --- echo "=== Building UI from source ===" cd "$DASHBOARD_DIR" -npm install --silent 2>/dev/null || true +# NOT silenced, and NOT `|| true`. Swallowing this is what turns a one-line +# EBADENGINE ("dashboard requires node >=24, you have v20") into the +# considerably less helpful "sh: next: command not found" from the build below, +# because the deps that provide `next` were never installed. +npm install npm run build # Copy the fresh build to the proxy's static UI directory cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/" @@ -188,9 +225,37 @@ PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAM # --- Playwright --- echo "=== Installing Playwright dependencies ===" cd "$SCRIPT_DIR" -npm install --silent 2>/dev/null || true +# Same reasoning as the dashboard install above: a failure here means the suite +# has no @playwright/test, and the run should say that rather than fail later. +npm install npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium +# Authoring a new spec means running it over and over against a stack that is +# already up -- rebuilding the UI and re-seeding for every iteration costs +# minutes each time. E2E_KEEP_ALIVE brings the stack up, then blocks, so you can +# run `npx playwright test ` yourself from another shell against it. +# Ctrl-C here tears everything down through the usual trap. +if [ "${E2E_KEEP_ALIVE:-0}" = "1" ]; then + cat < + +Press Ctrl-C to tear the stack down. +EOF + while kill -0 "$PROXY_PID" 2>/dev/null; do + sleep 5 + done + echo "Proxy exited." + exit 1 +fi + echo "=== Running Playwright tests ===" npx playwright test --config playwright.config.ts "$@" EXIT_CODE=$? diff --git a/tests/e2e/ui/tests/logs/logs.spec.ts b/tests/e2e/ui/tests/logs/logs.spec.ts new file mode 100644 index 00000000000..4c40afbfdeb --- /dev/null +++ b/tests/e2e/ui/tests/logs/logs.spec.ts @@ -0,0 +1,215 @@ +import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic"; + +/** + * Logs page manual-QA coverage: a request the proxy actually served shows up in + * the table, its detail drawer expands to the real request and response bodies, + * both can be copied, and the End User filter narrows the table to it. + * + * Every assertion is anchored to traffic this spec generates itself (unique + * prompt + unique end user per run), so it neither depends on seeded spend rows + * nor collides with the other specs' traffic when the suite runs in parallel. + */ + +const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + +/** + * The Input/Output cards in the drawer are built from SectionHeader, whose + * label is an antd Text span nested icon-div > flex-row > header-root. Walking + * up from the label is the only stable handle: the header has no role, test id + * or class of its own, and its copy button is icon-only (its "Copy" tooltip + * only exists while hovered, so it has no accessible name to query by). + */ +const sectionHeader = (drawer: Locator, label: "Input" | "Output"): Locator => + drawer.getByText(label, { exact: true }).locator("xpath=../../.."); + +/** + * The Logs page mounts every tab (Request Logs, Audit Logs, Deleted Keys, + * Deleted Teams), so the DOM holds four tables and four data-table toolbars at + * once and only the active tab's are visible. Unscoped `table tbody tr` counts + * rows from all four; every locator here goes through the visible one. + */ +const requestLogsRows = (page: PlaywrightPage): Locator => + page.locator("table").filter({ visible: true }).first().locator("tbody tr"); + +const visibleTestId = (page: PlaywrightPage, id: string): Locator => page.getByTestId(id).filter({ visible: true }); + +/** Open the Logs page and filter the table down to a single request id. */ +async function openLogsForRequest(page: PlaywrightPage, requestId: string): Promise { + await navigateToPage(page, Page.Logs); + await dismissFeedbackPopup(page); + + const search = visibleTestId(page, "datatable-search"); + await expect(search).toBeVisible({ timeout: 20_000 }); + await search.fill(requestId); + + const row = requestLogsRows(page).filter({ hasText: requestId }); + await expect(row, `no logs row for request ${requestId}`).toHaveCount(1, { + timeout: 30_000, + }); + return row; +} + +test.describe("Logs page", () => { + test.use({ + storageState: ADMIN_STORAGE_PATH, + // The drawer's copy buttons go through navigator.clipboard; without these + // the writes reject and the success toast never fires. + permissions: ["clipboard-read", "clipboard-write"], + }); + + test("a served request expands to its request and response, and both copy", async ({ page, request }) => { + const prompt = `logs-detail-prompt-${uniqueSuffix()}`; + const requestId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt, + }); + await waitForSpendLog(request, requestId); + + const row = await openLogsForRequest(page, requestId); + + // Expand: clicking the row opens the detail drawer for that request. + await row.click(); + const drawer = page.locator(".ant-drawer-content").first(); + await expect(drawer).toBeVisible({ timeout: 20_000 }); + await expect(drawer.getByText("Request & Response")).toBeVisible({ + timeout: 20_000, + }); + + // The prompt we sent and the mock server's reply are both rendered. + await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({ + timeout: 20_000, + }); + await expect(drawer.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 20_000 }); + + // Copy request: the Input card's copy button puts the prompt on the clipboard. + await sectionHeader(drawer, "Input").getByRole("button").click(); + await expect(page.getByText("Input copied")).toBeVisible({ + timeout: 10_000, + }); + expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(prompt); + + // Copy response: the Output card's copy button puts the completion on it. + await sectionHeader(drawer, "Output").getByRole("button").click(); + await expect(page.getByText("Output copied")).toBeVisible({ + timeout: 10_000, + }); + expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(MOCK_RESPONSE_TEXT); + }); + + test("the Input card collapses and expands", async ({ page, request }) => { + const prompt = `logs-collapse-prompt-${uniqueSuffix()}`; + const requestId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt, + }); + await waitForSpendLog(request, requestId); + + const row = await openLogsForRequest(page, requestId); + await row.click(); + + const drawer = page.locator(".ant-drawer-content").first(); + await expect(drawer.getByText("Request & Response")).toBeVisible({ + timeout: 20_000, + }); + + // SectionHeader renders an up-arrow while expanded and a down-arrow while + // collapsed. The body is the header's next sibling and collapses via + // `max-height: 0; overflow: hidden`, which zeroes its own bounding box — + // so the wrapper reads as hidden even though the prompt text node inside + // it does not (its box keeps its size, it is merely clipped by the parent). + const header = sectionHeader(drawer, "Input"); + const body = header.locator("xpath=following-sibling::div[1]"); + await expect(header.locator(".anticon-up")).toBeVisible(); + await expect(body).toBeVisible(); + + await header.click(); + await expect(header.locator(".anticon-down")).toBeVisible({ + timeout: 10_000, + }); + await expect(body).toBeHidden({ timeout: 10_000 }); + + await header.click(); + await expect(header.locator(".anticon-up")).toBeVisible({ + timeout: 10_000, + }); + await expect(body).toBeVisible({ timeout: 10_000 }); + await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({ + timeout: 10_000, + }); + }); + + test("the JSON view exposes Request and Response tabs", async ({ page, request }) => { + const prompt = `logs-json-prompt-${uniqueSuffix()}`; + const requestId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt, + }); + await waitForSpendLog(request, requestId); + + const row = await openLogsForRequest(page, requestId); + await row.click(); + + const drawer = page.locator(".ant-drawer-content").first(); + await expect(drawer.getByText("Request & Response")).toBeVisible({ + timeout: 20_000, + }); + + // antd Radio.Button hides the under its