mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
test(e2e-ui): cover the Playground, Logs and Usage manual-QA flows
These three pages carried no e2e coverage, so the manual QA checklist was the only thing standing behind them. Playground: sends a chat from the UI for both configured models, and for both virtual-key sources (the logged-in session, and a key pasted into the panel). This is the only spec that drives the dashboard's own LLM call path rather than an admin CRUD endpoint. Logs: a request the proxy actually served appears in the table, its drawer expands to the real request and response bodies, both copy to the clipboard, the Input card collapses, the JSON view exposes Request/Response, and the End User filter narrows the table to one customer. Usage: traffic billed to a virtual key reaches Top Virtual Keys, the card toggles between table and chart, and the key opens its key-info panel. Router settings: the existing spec proved the UI can record a fallback; the new one proves the fallback is honoured, by pointing a model at an unreachable upstream and asserting the reply comes back anyway. It asserts the un-fallen-back call fails first, so a quietly-working primary cannot fake a pass. Supporting changes: - helpers/traffic.ts generates the traffic these pages render, rather than seeding rows no code produced. Its two wait helpers exist because the Logs and Usage pages read different stores: spend logs are flushed on a timer, and the Usage page reads a background rollup *and* fetches once on mount, so waiting on the DOM there can never converge. - helpers/playground.ts holds the playground controls, now shared with the fallback spec. Everything is scoped to the visible copy of the config panel, which is rendered twice for the docked and collapsed layouts. - run_e2e.sh gains E2E_KEEP_ALIVE=1, which brings the stack up and blocks so a spec can be re-run against it without paying for a UI rebuild each iteration. Verified with the full suite on a fresh stack: 89 passed, 0 failed, 5 skipped.
This commit is contained in:
Yuneng Jiang
parent
e24a9146e3
commit
30968fb721
7 changed files with 753 additions and 9 deletions
57
tests/e2e/ui/helpers/playground.ts
Normal file
57
tests/e2e/ui/helpers/playground.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "./navigation";
|
||||
import { Page } from "../fixtures/pages";
|
||||
|
||||
/**
|
||||
* Controls for the Test Key / Playground page.
|
||||
*
|
||||
* Shared because more than one manual-QA flow ends in "send a message from the
|
||||
* UI and check what comes back" — the playground itself, and router fallbacks,
|
||||
* which are only really verified by seeing the fallback's reply render.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The playground renders its configuration panel twice — once for the docked
|
||||
* sidebar and once for the collapsed/overlay layout — and only one copy is on
|
||||
* screen at a time. Every control here is narrowed to the visible copy; an
|
||||
* unscoped locator hits a strict-mode violation against its hidden twin.
|
||||
*/
|
||||
export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first();
|
||||
|
||||
/** The model dropdown, addressed by the placeholder it shows before selection. */
|
||||
export const modelSelect = (page: PlaywrightPage): Locator =>
|
||||
onlyVisible(page.locator('.ant-select:has(.ant-select-selection-placeholder:text-is("Select a Model"))'));
|
||||
|
||||
/** Send button is icon-only (an up-arrow), so there is no accessible name. */
|
||||
export const sendButton = (page: PlaywrightPage): Locator => onlyVisible(page.locator("button:has(.anticon-arrow-up)"));
|
||||
|
||||
/** The Virtual Key Source dropdown, addressed by its currently selected label. */
|
||||
export const keySourceSelect = (page: PlaywrightPage, current: string): Locator =>
|
||||
onlyVisible(page.locator(`.ant-select:has(.ant-select-selection-item[title="${current}"])`));
|
||||
|
||||
export async function openPlayground(page: PlaywrightPage): Promise<void> {
|
||||
await navigateToPage(page, Page.LlmPlayground);
|
||||
await dismissFeedbackPopup(page);
|
||||
await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
}
|
||||
|
||||
export async function selectModel(page: PlaywrightPage, model: string): Promise<void> {
|
||||
const select = modelSelect(page);
|
||||
await select.click();
|
||||
// The dropdown is virtualized: only the options in the rendered window exist
|
||||
// in the DOM, so once other specs have added models to the proxy the one we
|
||||
// want is not merely off-screen, it is absent. The Select takes showSearch,
|
||||
// so type to narrow the list before clicking.
|
||||
await select.locator("input.ant-select-selection-search-input").fill(model);
|
||||
// antd portals its dropdown to the body; options carry the value as `title`.
|
||||
await onlyVisible(page.locator(`.ant-select-item-option[title="${model}"]`)).click({ timeout: 15_000 });
|
||||
}
|
||||
|
||||
export async function sendMessage(page: PlaywrightPage, message: string): Promise<void> {
|
||||
const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false }));
|
||||
await expect(input).toBeVisible({ timeout: 15_000 });
|
||||
await input.fill(message);
|
||||
await sendButton(page).click();
|
||||
}
|
||||
153
tests/e2e/ui/helpers/traffic.ts
Normal file
153
tests/e2e/ui/helpers/traffic.ts
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
import { APIRequestContext, expect } from "@playwright/test";
|
||||
|
||||
/**
|
||||
* Helpers for driving real traffic through the proxy from a test.
|
||||
*
|
||||
* Several manual-QA flows (Logs, Usage) can only be checked once the proxy has
|
||||
* actually served a request: the spend log a Logs row renders, and the
|
||||
* per-key spend the Usage page aggregates, are both written by the completion
|
||||
* path. Seeding those tables directly would test the UI against rows no code
|
||||
* produced, so these helpers make the proxy generate them the same way a user
|
||||
* would.
|
||||
*/
|
||||
|
||||
/** Model names served by fixtures/config.yml, both backed by the mock LLM server. */
|
||||
export const CHAT_MODEL_A = "fake-openai-gpt-4";
|
||||
export const CHAT_MODEL_B = "fake-anthropic-claude";
|
||||
|
||||
/** The only completion text fixtures/mock_llm_server/server.py ever returns. */
|
||||
export const MOCK_RESPONSE_TEXT = "This is a mock response.";
|
||||
|
||||
export const masterKey = (): string => process.env.LITELLM_MASTER_KEY || "sk-1234";
|
||||
|
||||
/** Root path the proxy is mounted under, "" for the default mount. */
|
||||
const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? "";
|
||||
|
||||
interface ChatOptions {
|
||||
model: string;
|
||||
prompt: string;
|
||||
/** Virtual key to bill the call to. Defaults to the master key. */
|
||||
apiKey?: string;
|
||||
/** Sent as `user`, which lands in the spend log's end_user column. */
|
||||
endUser?: string;
|
||||
}
|
||||
|
||||
/** POST /v1/chat/completions and return the completion id (the Logs Request ID). */
|
||||
export async function sendChatCompletion(request: APIRequestContext, opts: ChatOptions): Promise<string> {
|
||||
const res = await request.post(`${rootPath()}/v1/chat/completions`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${opts.apiKey ?? masterKey()}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
model: opts.model,
|
||||
messages: [{ role: "user", content: opts.prompt }],
|
||||
...(opts.endUser ? { user: opts.endUser } : {}),
|
||||
},
|
||||
});
|
||||
expect(res.ok(), `chat completion for ${opts.model} failed (${res.status()}): ${await res.text()}`).toBe(true);
|
||||
const body = await res.json();
|
||||
expect(body.choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT);
|
||||
return body.id as string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a virtual key via /key/generate.
|
||||
*
|
||||
* `key` is the sk- value callers authenticate with; `token` is its hash, which
|
||||
* is what usage/spend aggregates are keyed by (the plaintext key is never
|
||||
* stored, so it never appears in those responses).
|
||||
*/
|
||||
export async function createVirtualKey(
|
||||
request: APIRequestContext,
|
||||
data: Record<string, unknown> = {},
|
||||
): Promise<{ key: string; token: string; alias?: string }> {
|
||||
const res = await request.post(`${rootPath()}/key/generate`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${masterKey()}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data,
|
||||
});
|
||||
expect(res.ok(), `key generate failed (${res.status()}): ${await res.text()}`).toBe(true);
|
||||
const body = await res.json();
|
||||
return {
|
||||
key: body.key as string,
|
||||
token: (body.token ?? body.token_id) as string,
|
||||
alias: body.key_alias as string | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Spend logs are flushed on a timer, not synchronously with the response, so a
|
||||
* Logs/Usage assertion made straight after a completion races the writer. Poll
|
||||
* /spend/logs until the request id shows up rather than sleeping a fixed amount.
|
||||
*/
|
||||
export async function waitForSpendLog(
|
||||
request: APIRequestContext,
|
||||
requestId: string,
|
||||
timeoutMs = 60_000,
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastStatus = 0;
|
||||
while (Date.now() < deadline) {
|
||||
const res = await request.get(`${rootPath()}/spend/logs?request_id=${encodeURIComponent(requestId)}`, {
|
||||
headers: { Authorization: `Bearer ${masterKey()}` },
|
||||
});
|
||||
lastStatus = res.status();
|
||||
if (res.ok()) {
|
||||
const body = await res.json();
|
||||
const rows = Array.isArray(body) ? body : body?.data ?? [];
|
||||
if (rows.length > 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 2_000));
|
||||
}
|
||||
throw new Error(`spend log for request ${requestId} never appeared (last /spend/logs status ${lastStatus})`);
|
||||
}
|
||||
|
||||
const isoDay = (d: Date): string => d.toISOString().slice(0, 10);
|
||||
|
||||
/**
|
||||
* Wait until a key's traffic has been rolled up into the daily-spend aggregate.
|
||||
*
|
||||
* The Usage page is built from /user/daily/activity, which reads the aggregate
|
||||
* table a background job writes — not the spend log itself. It also fetches
|
||||
* once on mount and never refetches, so a test that navigates before the
|
||||
* rollup lands will keep re-reading a stale render until it times out. Waiting
|
||||
* on the API first is the only way to make that deterministic.
|
||||
*/
|
||||
export async function waitForKeyInDailyActivity(
|
||||
request: APIRequestContext,
|
||||
keyToken: string,
|
||||
timeoutMs = 120_000,
|
||||
): Promise<void> {
|
||||
const now = new Date();
|
||||
const start = new Date(now);
|
||||
start.setDate(start.getDate() - 7);
|
||||
const query = `start_date=${isoDay(start)}&end_date=${isoDay(now)}`;
|
||||
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastStatus = 0;
|
||||
while (Date.now() < deadline) {
|
||||
const res = await request.get(`${rootPath()}/user/daily/activity?${query}`, {
|
||||
headers: { Authorization: `Bearer ${masterKey()}` },
|
||||
});
|
||||
lastStatus = res.status();
|
||||
if (res.ok()) {
|
||||
const body = await res.json();
|
||||
const seen = (body?.results ?? []).some(
|
||||
(day: { breakdown?: { api_keys?: Record<string, unknown> } }) => keyToken in (day.breakdown?.api_keys ?? {}),
|
||||
);
|
||||
if (seen) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 3_000));
|
||||
}
|
||||
throw new Error(
|
||||
`key ${keyToken} never appeared in /user/daily/activity (last status ${lastStatus}); ` +
|
||||
"the daily spend rollup may not be running",
|
||||
);
|
||||
}
|
||||
|
|
@ -33,7 +33,35 @@ if [ "$IS_CI" = "false" ]; then
|
|||
for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do
|
||||
[ -d "$p" ] && export PATH="$p:$PATH"
|
||||
done
|
||||
[ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh"
|
||||
# Sourcing nvm only makes `nvm` available -- it leaves you on whatever the
|
||||
# default alias points at, which is frequently an older Node than the
|
||||
# dashboard's engines allow. `npm install` then fails EBADENGINE, npm exits
|
||||
# non-zero, and because the install below is `--silent ... || true` the error
|
||||
# is swallowed and the run dies later with the far less obvious
|
||||
# "sh: next: command not found".
|
||||
#
|
||||
# So select a Node that satisfies ui/litellm-dashboard's engines.node, and if
|
||||
# none is available say so here rather than 200 lines downstream.
|
||||
if [ -s "$HOME/.nvm/nvm.sh" ]; then
|
||||
# shellcheck disable=SC1091
|
||||
source "$HOME/.nvm/nvm.sh"
|
||||
required_major="$(sed -nE 's/.*"node"[[:space:]]*:[[:space:]]*">=?([0-9]+).*/\1/p' \
|
||||
"$DASHBOARD_DIR/package.json" 2>/dev/null | head -1)"
|
||||
if [ -n "$required_major" ]; then
|
||||
current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')"
|
||||
if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then
|
||||
echo "Node $(node --version 2>/dev/null || echo 'not found') is below the dashboard's required v${required_major}; selecting a newer one via nvm"
|
||||
nvm use "$required_major" >/dev/null 2>&1 || nvm use --lts >/dev/null 2>&1 || true
|
||||
current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')"
|
||||
if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then
|
||||
echo "Error: ui/litellm-dashboard requires Node >= v${required_major}, and no such version is installed."
|
||||
echo " Install one with: nvm install ${required_major}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo "Using Node $(node --version) / npm $(npm --version)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Cleanup on exit ---
|
||||
|
|
@ -59,8 +87,13 @@ if [ "$IS_CI" = "false" ]; then
|
|||
for cmd in docker psql; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
|
||||
done
|
||||
# Only a LISTENER conflicts with us. Without -sTCP:LISTEN this also matches
|
||||
# ESTABLISHED sockets, so an unrelated *outbound* connection from this machine
|
||||
# to someone else's :5432 (a psql session, a running app, a Prisma engine
|
||||
# talking to a remote database) aborts the run with "port 5432 is in use"
|
||||
# while nothing is actually bound locally.
|
||||
for port in 4000 5432 8090; do
|
||||
if lsof -ti ":$port" >/dev/null 2>&1; then
|
||||
if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
|
||||
echo "Error: port $port is in use"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -108,7 +141,11 @@ export LITELLM_LICENSE="${LITELLM_LICENSE:-}"
|
|||
# --- Rebuild UI from source ---
|
||||
echo "=== Building UI from source ==="
|
||||
cd "$DASHBOARD_DIR"
|
||||
npm install --silent 2>/dev/null || true
|
||||
# NOT silenced, and NOT `|| true`. Swallowing this is what turns a one-line
|
||||
# EBADENGINE ("dashboard requires node >=24, you have v20") into the
|
||||
# considerably less helpful "sh: next: command not found" from the build below,
|
||||
# because the deps that provide `next` were never installed.
|
||||
npm install
|
||||
npm run build
|
||||
# Copy the fresh build to the proxy's static UI directory
|
||||
cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/"
|
||||
|
|
@ -188,9 +225,37 @@ PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAM
|
|||
# --- Playwright ---
|
||||
echo "=== Installing Playwright dependencies ==="
|
||||
cd "$SCRIPT_DIR"
|
||||
npm install --silent 2>/dev/null || true
|
||||
# Same reasoning as the dashboard install above: a failure here means the suite
|
||||
# has no @playwright/test, and the run should say that rather than fail later.
|
||||
npm install
|
||||
npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium
|
||||
|
||||
# Authoring a new spec means running it over and over against a stack that is
|
||||
# already up -- rebuilding the UI and re-seeding for every iteration costs
|
||||
# minutes each time. E2E_KEEP_ALIVE brings the stack up, then blocks, so you can
|
||||
# run `npx playwright test <spec>` yourself from another shell against it.
|
||||
# Ctrl-C here tears everything down through the usual trap.
|
||||
if [ "${E2E_KEEP_ALIVE:-0}" = "1" ]; then
|
||||
cat <<EOF
|
||||
|
||||
=== Stack is up (E2E_KEEP_ALIVE=1); not running tests ===
|
||||
UI / API : http://127.0.0.1:4000
|
||||
Mock LLM : http://127.0.0.1:8090/v1
|
||||
Database : $DATABASE_URL
|
||||
Proxy log: $PROXY_LOG
|
||||
|
||||
Run specs against it from $SCRIPT_DIR:
|
||||
npx playwright test --config playwright.config.ts <spec>
|
||||
|
||||
Press Ctrl-C to tear the stack down.
|
||||
EOF
|
||||
while kill -0 "$PROXY_PID" 2>/dev/null; do
|
||||
sleep 5
|
||||
done
|
||||
echo "Proxy exited."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Running Playwright tests ==="
|
||||
npx playwright test --config playwright.config.ts "$@"
|
||||
EXIT_CODE=$?
|
||||
|
|
|
|||
215
tests/e2e/ui/tests/logs/logs.spec.ts
Normal file
215
tests/e2e/ui/tests/logs/logs.spec.ts
Normal file
|
|
@ -0,0 +1,215 @@
|
|||
import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic";
|
||||
|
||||
/**
|
||||
* Logs page manual-QA coverage: a request the proxy actually served shows up in
|
||||
* the table, its detail drawer expands to the real request and response bodies,
|
||||
* both can be copied, and the End User filter narrows the table to it.
|
||||
*
|
||||
* Every assertion is anchored to traffic this spec generates itself (unique
|
||||
* prompt + unique end user per run), so it neither depends on seeded spend rows
|
||||
* nor collides with the other specs' traffic when the suite runs in parallel.
|
||||
*/
|
||||
|
||||
const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
|
||||
/**
|
||||
* The Input/Output cards in the drawer are built from SectionHeader, whose
|
||||
* label is an antd Text span nested icon-div > flex-row > header-root. Walking
|
||||
* up from the label is the only stable handle: the header has no role, test id
|
||||
* or class of its own, and its copy button is icon-only (its "Copy" tooltip
|
||||
* only exists while hovered, so it has no accessible name to query by).
|
||||
*/
|
||||
const sectionHeader = (drawer: Locator, label: "Input" | "Output"): Locator =>
|
||||
drawer.getByText(label, { exact: true }).locator("xpath=../../..");
|
||||
|
||||
/**
|
||||
* The Logs page mounts every tab (Request Logs, Audit Logs, Deleted Keys,
|
||||
* Deleted Teams), so the DOM holds four tables and four data-table toolbars at
|
||||
* once and only the active tab's are visible. Unscoped `table tbody tr` counts
|
||||
* rows from all four; every locator here goes through the visible one.
|
||||
*/
|
||||
const requestLogsRows = (page: PlaywrightPage): Locator =>
|
||||
page.locator("table").filter({ visible: true }).first().locator("tbody tr");
|
||||
|
||||
const visibleTestId = (page: PlaywrightPage, id: string): Locator => page.getByTestId(id).filter({ visible: true });
|
||||
|
||||
/** Open the Logs page and filter the table down to a single request id. */
|
||||
async function openLogsForRequest(page: PlaywrightPage, requestId: string): Promise<Locator> {
|
||||
await navigateToPage(page, Page.Logs);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
const search = visibleTestId(page, "datatable-search");
|
||||
await expect(search).toBeVisible({ timeout: 20_000 });
|
||||
await search.fill(requestId);
|
||||
|
||||
const row = requestLogsRows(page).filter({ hasText: requestId });
|
||||
await expect(row, `no logs row for request ${requestId}`).toHaveCount(1, {
|
||||
timeout: 30_000,
|
||||
});
|
||||
return row;
|
||||
}
|
||||
|
||||
test.describe("Logs page", () => {
|
||||
test.use({
|
||||
storageState: ADMIN_STORAGE_PATH,
|
||||
// The drawer's copy buttons go through navigator.clipboard; without these
|
||||
// the writes reject and the success toast never fires.
|
||||
permissions: ["clipboard-read", "clipboard-write"],
|
||||
});
|
||||
|
||||
test("a served request expands to its request and response, and both copy", async ({ page, request }) => {
|
||||
const prompt = `logs-detail-prompt-${uniqueSuffix()}`;
|
||||
const requestId = await sendChatCompletion(request, {
|
||||
model: CHAT_MODEL_A,
|
||||
prompt,
|
||||
});
|
||||
await waitForSpendLog(request, requestId);
|
||||
|
||||
const row = await openLogsForRequest(page, requestId);
|
||||
|
||||
// Expand: clicking the row opens the detail drawer for that request.
|
||||
await row.click();
|
||||
const drawer = page.locator(".ant-drawer-content").first();
|
||||
await expect(drawer).toBeVisible({ timeout: 20_000 });
|
||||
await expect(drawer.getByText("Request & Response")).toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
|
||||
// The prompt we sent and the mock server's reply are both rendered.
|
||||
await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
await expect(drawer.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 20_000 });
|
||||
|
||||
// Copy request: the Input card's copy button puts the prompt on the clipboard.
|
||||
await sectionHeader(drawer, "Input").getByRole("button").click();
|
||||
await expect(page.getByText("Input copied")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(prompt);
|
||||
|
||||
// Copy response: the Output card's copy button puts the completion on it.
|
||||
await sectionHeader(drawer, "Output").getByRole("button").click();
|
||||
await expect(page.getByText("Output copied")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(MOCK_RESPONSE_TEXT);
|
||||
});
|
||||
|
||||
test("the Input card collapses and expands", async ({ page, request }) => {
|
||||
const prompt = `logs-collapse-prompt-${uniqueSuffix()}`;
|
||||
const requestId = await sendChatCompletion(request, {
|
||||
model: CHAT_MODEL_A,
|
||||
prompt,
|
||||
});
|
||||
await waitForSpendLog(request, requestId);
|
||||
|
||||
const row = await openLogsForRequest(page, requestId);
|
||||
await row.click();
|
||||
|
||||
const drawer = page.locator(".ant-drawer-content").first();
|
||||
await expect(drawer.getByText("Request & Response")).toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
|
||||
// SectionHeader renders an up-arrow while expanded and a down-arrow while
|
||||
// collapsed. The body is the header's next sibling and collapses via
|
||||
// `max-height: 0; overflow: hidden`, which zeroes its own bounding box —
|
||||
// so the wrapper reads as hidden even though the prompt text node inside
|
||||
// it does not (its box keeps its size, it is merely clipped by the parent).
|
||||
const header = sectionHeader(drawer, "Input");
|
||||
const body = header.locator("xpath=following-sibling::div[1]");
|
||||
await expect(header.locator(".anticon-up")).toBeVisible();
|
||||
await expect(body).toBeVisible();
|
||||
|
||||
await header.click();
|
||||
await expect(header.locator(".anticon-down")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await expect(body).toBeHidden({ timeout: 10_000 });
|
||||
|
||||
await header.click();
|
||||
await expect(header.locator(".anticon-up")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await expect(body).toBeVisible({ timeout: 10_000 });
|
||||
await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
});
|
||||
|
||||
test("the JSON view exposes Request and Response tabs", async ({ page, request }) => {
|
||||
const prompt = `logs-json-prompt-${uniqueSuffix()}`;
|
||||
const requestId = await sendChatCompletion(request, {
|
||||
model: CHAT_MODEL_A,
|
||||
prompt,
|
||||
});
|
||||
await waitForSpendLog(request, requestId);
|
||||
|
||||
const row = await openLogsForRequest(page, requestId);
|
||||
await row.click();
|
||||
|
||||
const drawer = page.locator(".ant-drawer-content").first();
|
||||
await expect(drawer.getByText("Request & Response")).toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
|
||||
// antd Radio.Button hides the <input> under its <label>, which intercepts
|
||||
// the pointer event — click the label, not the radio.
|
||||
await drawer.locator("label.ant-radio-button-wrapper").filter({ hasText: "JSON" }).click();
|
||||
|
||||
const requestTab = drawer.getByRole("tab", { name: "Request" });
|
||||
await expect(requestTab).toBeVisible({ timeout: 10_000 });
|
||||
await requestTab.click();
|
||||
await expect(drawer.getByText(prompt, { exact: false }).first()).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await drawer.getByRole("tab", { name: "Response" }).click();
|
||||
await expect(drawer.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 10_000 });
|
||||
});
|
||||
|
||||
test("the End User filter narrows the table to that customer", async ({ page, request }) => {
|
||||
const endUser = `logs-end-user-${uniqueSuffix()}`;
|
||||
const minePrompt = `logs-filter-mine-${uniqueSuffix()}`;
|
||||
const otherPrompt = `logs-filter-other-${uniqueSuffix()}`;
|
||||
|
||||
const mineId = await sendChatCompletion(request, {
|
||||
model: CHAT_MODEL_A,
|
||||
prompt: minePrompt,
|
||||
endUser,
|
||||
});
|
||||
const otherId = await sendChatCompletion(request, {
|
||||
model: CHAT_MODEL_A,
|
||||
prompt: otherPrompt,
|
||||
});
|
||||
await waitForSpendLog(request, mineId);
|
||||
await waitForSpendLog(request, otherId);
|
||||
|
||||
await navigateToPage(page, Page.Logs);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
// Both requests are in the unfiltered table.
|
||||
await expect(requestLogsRows(page).filter({ hasText: mineId })).toHaveCount(1, { timeout: 30_000 });
|
||||
await expect(requestLogsRows(page).filter({ hasText: otherId })).toHaveCount(1, { timeout: 30_000 });
|
||||
|
||||
await visibleTestId(page, "datatable-filters-trigger").click();
|
||||
const filters = page.getByRole("dialog").filter({ hasText: "Narrow down request logs" });
|
||||
await expect(filters).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
const endUserInput = filters.getByPlaceholder("Search an end user");
|
||||
await endUserInput.click();
|
||||
await endUserInput.fill(endUser);
|
||||
// The combobox popup is portaled to the body, so it is outside the filter
|
||||
// dialog's subtree — scope the option lookup to the page, not the dialog.
|
||||
await page.getByRole("option", { name: endUser, exact: true }).click({ timeout: 30_000 });
|
||||
await filters.getByRole("button", { name: "Apply Filters" }).click();
|
||||
|
||||
// Only the request tagged with this end user survives the filter.
|
||||
await expect(requestLogsRows(page).filter({ hasText: otherId })).toHaveCount(0, { timeout: 30_000 });
|
||||
await expect(requestLogsRows(page).filter({ hasText: mineId })).toHaveCount(1);
|
||||
await expect(requestLogsRows(page)).toHaveCount(1);
|
||||
});
|
||||
});
|
||||
56
tests/e2e/ui/tests/playground/playground.spec.ts
Normal file
56
tests/e2e/ui/tests/playground/playground.spec.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import { test, expect } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { CHAT_MODEL_A, CHAT_MODEL_B, MOCK_RESPONSE_TEXT, createVirtualKey } from "../../helpers/traffic";
|
||||
import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground";
|
||||
|
||||
/**
|
||||
* Test Key / Playground manual-QA coverage: a chat sent from the UI reaches the
|
||||
* proxy and renders the model's reply, for both configured models and for both
|
||||
* virtual-key sources (the logged-in UI session, and a virtual key pasted in).
|
||||
*
|
||||
* This is the one flow that exercises the dashboard's own LLM call path rather
|
||||
* than an admin CRUD endpoint, so it is the check that the UI's request
|
||||
* plumbing (auth header, endpoint selection, streaming render) still works.
|
||||
*/
|
||||
test.describe("Playground", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
for (const model of [CHAT_MODEL_A, CHAT_MODEL_B]) {
|
||||
test(`chats with ${model} using the current UI session`, async ({ page }) => {
|
||||
await openPlayground(page);
|
||||
|
||||
// "Current UI Session" is the default source — the logged-in admin's key
|
||||
// is used, with no key pasted anywhere.
|
||||
await expect(onlyVisible(page.getByTitle("Current UI Session"))).toBeVisible();
|
||||
|
||||
await selectModel(page, model);
|
||||
const prompt = `playground ping for ${model}`;
|
||||
await sendMessage(page, prompt);
|
||||
|
||||
// Our prompt is echoed into the transcript, and the mock server replies.
|
||||
await expect(page.getByText(prompt, { exact: false }).first()).toBeVisible({ timeout: 20_000 });
|
||||
await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 });
|
||||
});
|
||||
}
|
||||
|
||||
test("chats using a pasted virtual key instead of the UI session", async ({ page, request }) => {
|
||||
const { key } = await createVirtualKey(request, {
|
||||
key_alias: `e2e-playground-${Date.now()}`,
|
||||
});
|
||||
|
||||
await openPlayground(page);
|
||||
|
||||
// Switch the source to "Virtual Key" and paste the key we just minted.
|
||||
await keySourceSelect(page, "Current UI Session").click();
|
||||
await onlyVisible(page.locator('.ant-select-item-option[title="Virtual Key"]')).click({ timeout: 15_000 });
|
||||
|
||||
const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
|
||||
await expect(keyInput).toBeVisible({ timeout: 10_000 });
|
||||
await keyInput.fill(key);
|
||||
|
||||
await selectModel(page, CHAT_MODEL_A);
|
||||
await sendMessage(page, "playground ping via virtual key");
|
||||
|
||||
await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 });
|
||||
});
|
||||
});
|
||||
|
|
@ -3,6 +3,8 @@ import { ADMIN_STORAGE_PATH } from "../../constants";
|
|||
import { navigateToPage } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { Role, users } from "../../fixtures/users";
|
||||
import { MOCK_RESPONSE_TEXT } from "../../helpers/traffic";
|
||||
import { openPlayground, selectModel, sendMessage } from "../../helpers/playground";
|
||||
// Type-only import of the OpenAPI-generated backend schema, erased at runtime by
|
||||
// esbuild. It types the round-trips below so mistakes surface in the editor; the live
|
||||
// test against the real proxy is what actually enforces the contract.
|
||||
|
|
@ -79,7 +81,9 @@ test.describe("Router Settings - Fallbacks", () => {
|
|||
await primarySelect.click();
|
||||
await page.keyboard.type(PRIMARY);
|
||||
await page.keyboard.press("Enter");
|
||||
await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({ timeout: 10_000 });
|
||||
await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
const fallbackSelect = modal.locator(".ant-select").filter({ hasText: "Select fallback models" });
|
||||
await fallbackSelect.click();
|
||||
|
|
@ -88,7 +92,9 @@ test.describe("Router Settings - Fallbacks", () => {
|
|||
await page.keyboard.press("Escape");
|
||||
// The Fallback Chain helper text reads "(N/10 used)"; once it ticks to 1 the
|
||||
// selection has been recorded.
|
||||
await expect(modal.getByText("(1/10 used)")).toBeVisible({ timeout: 10_000 });
|
||||
await expect(modal.getByText("(1/10 used)")).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
// Save
|
||||
await modal.getByRole("button", { name: /Save All Configurations/i }).click();
|
||||
|
|
@ -111,7 +117,9 @@ test.describe("Router Settings - Fallbacks", () => {
|
|||
type ConfigYAML = components["schemas"]["ConfigYAML"];
|
||||
type RouterSettingsResponse = components["schemas"]["RouterSettingsResponse"];
|
||||
|
||||
const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` };
|
||||
const ADMIN_AUTH = {
|
||||
Authorization: `Bearer ${users[Role.ProxyAdmin].password}`,
|
||||
};
|
||||
|
||||
/**
|
||||
* Apply a router_settings patch through the typed /config/update contract. The
|
||||
|
|
@ -172,13 +180,17 @@ test.describe("Router Settings - Loadbalancing", () => {
|
|||
// The ticket's core symptom was that a refresh showed the old value.
|
||||
await navigateToPage(page, Page.RouterSettings);
|
||||
await page.getByRole("tab", { name: "Loadbalancing" }).click();
|
||||
await expect(page.locator('input[name="num_retries"]')).toHaveValue("5", { timeout: 15_000 });
|
||||
await expect(page.locator('input[name="num_retries"]')).toHaveValue("5", {
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
// The typed backend read agrees the change persisted.
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const res = await request.get(`/router/settings`, { headers: ADMIN_AUTH });
|
||||
const res = await request.get(`/router/settings`, {
|
||||
headers: ADMIN_AUTH,
|
||||
});
|
||||
const data = (await res.json()) as RouterSettingsResponse;
|
||||
return data.current_values?.num_retries;
|
||||
},
|
||||
|
|
@ -187,3 +199,99 @@ test.describe("Router Settings - Loadbalancing", () => {
|
|||
.toBe(5);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The fallback test above proves the UI can *record* a fallback. This one
|
||||
* proves the recorded fallback is actually honoured: a model whose upstream is
|
||||
* unreachable still answers, and the answer is the fallback model's.
|
||||
*
|
||||
* The primary is created here rather than taken from fixtures/config.yml
|
||||
* because every configured model is backed by the mock server and therefore
|
||||
* healthy — there is nothing in the fixture set that can fail on demand.
|
||||
*/
|
||||
test.describe("Router Settings - Fallbacks serve the request", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
const BROKEN_PRIMARY = "e2e-broken-primary";
|
||||
let brokenModelId: string | null = null;
|
||||
|
||||
/** Drop only this test's fallback entry, leaving any others untouched. */
|
||||
async function clearBrokenFallback(request: import("@playwright/test").APIRequestContext) {
|
||||
const current = await request.get("/get/config/callbacks", {
|
||||
headers: ADMIN_AUTH,
|
||||
});
|
||||
if (!current.ok()) return;
|
||||
const router = (await current.json())?.router_settings ?? {};
|
||||
const existing: Array<Record<string, string[]>> = Array.isArray(router.fallbacks) ? router.fallbacks : [];
|
||||
await patchRouterSettings(request, {
|
||||
fallbacks: existing.filter((entry) => !(entry && BROKEN_PRIMARY in entry)),
|
||||
} as Partial<NonNullable<ConfigYAML["router_settings"]>>);
|
||||
}
|
||||
|
||||
test.beforeEach(async ({ request }) => {
|
||||
await clearBrokenFallback(request);
|
||||
|
||||
// Port 9 is the discard service: nothing listens, so the connection is
|
||||
// refused immediately rather than hanging until a timeout.
|
||||
const res = await request.post("/model/new", {
|
||||
headers: ADMIN_AUTH,
|
||||
data: {
|
||||
model_name: BROKEN_PRIMARY,
|
||||
litellm_params: {
|
||||
model: "openai/broken",
|
||||
api_base: "http://127.0.0.1:9/v1",
|
||||
api_key: "fake",
|
||||
timeout: 5,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(res.ok(), `creating the broken primary failed: ${res.status()} ${await res.text()}`).toBeTruthy();
|
||||
brokenModelId = (await res.json())?.model_id ?? null;
|
||||
});
|
||||
|
||||
test.afterEach(async ({ request }) => {
|
||||
await clearBrokenFallback(request);
|
||||
if (brokenModelId) {
|
||||
await request.post("/model/delete", {
|
||||
headers: ADMIN_AUTH,
|
||||
data: { id: brokenModelId },
|
||||
});
|
||||
brokenModelId = null;
|
||||
}
|
||||
});
|
||||
|
||||
test("a request to an unreachable model is answered by its fallback", async ({ page, request }) => {
|
||||
const chat = async () =>
|
||||
request.post("/v1/chat/completions", {
|
||||
headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
|
||||
data: {
|
||||
model: BROKEN_PRIMARY,
|
||||
messages: [{ role: "user", content: "fallback probe" }],
|
||||
},
|
||||
});
|
||||
|
||||
// Without a fallback the primary's failure is the client's failure. This is
|
||||
// the control: it proves the mock reply asserted below could only have come
|
||||
// from the fallback, not from the primary quietly working.
|
||||
expect((await chat()).status(), "broken primary unexpectedly succeeded on its own").toBeGreaterThanOrEqual(400);
|
||||
|
||||
await patchRouterSettings(request, {
|
||||
fallbacks: [{ [BROKEN_PRIMARY]: [PRIMARY] }],
|
||||
} as Partial<NonNullable<ConfigYAML["router_settings"]>>);
|
||||
|
||||
// Same call now succeeds, served by the fallback model.
|
||||
await expect
|
||||
.poll(async () => (await chat()).status(), {
|
||||
timeout: 30_000,
|
||||
message: "fallback never took effect",
|
||||
})
|
||||
.toBe(200);
|
||||
|
||||
// And the UI shows it: the playground renders a reply for a model whose own
|
||||
// upstream is down.
|
||||
await openPlayground(page);
|
||||
await selectModel(page, BROKEN_PRIMARY);
|
||||
await sendMessage(page, "fallback probe from the playground");
|
||||
await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
90
tests/e2e/ui/tests/usage/usagePage.spec.ts
Normal file
90
tests/e2e/ui/tests/usage/usagePage.spec.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import {
|
||||
CHAT_MODEL_A,
|
||||
createVirtualKey,
|
||||
sendChatCompletion,
|
||||
waitForKeyInDailyActivity,
|
||||
waitForSpendLog,
|
||||
} from "../../helpers/traffic";
|
||||
|
||||
/**
|
||||
* Usage page manual-QA coverage: traffic billed to a virtual key shows up in
|
||||
* Top Virtual Keys, the card switches between its table and chart renderings,
|
||||
* and clicking the key opens its key-info panel.
|
||||
*
|
||||
* Targets the current Usage page (/ui/usage). The legacy /ui/old-usage view
|
||||
* carries its own deprecation banner and is deliberately not covered here.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The Top Virtual Keys card. Its <Title> is a direct child of the card element,
|
||||
* so stepping up one level from the title is an exact handle — needed because
|
||||
* the page renders several other tables (Spend by Provider, Top Models) that an
|
||||
* unscoped table locator would pick up.
|
||||
*/
|
||||
const topKeysCard = (page: PlaywrightPage): Locator =>
|
||||
page.getByText("Top Virtual Keys", { exact: true }).locator("xpath=..");
|
||||
|
||||
async function openUsage(page: PlaywrightPage): Promise<Locator> {
|
||||
await navigateToPage(page, Page.NewUsage);
|
||||
await dismissFeedbackPopup(page);
|
||||
const card = topKeysCard(page);
|
||||
await expect(card).toBeVisible({ timeout: 30_000 });
|
||||
// Widen the leaderboard so the key under test is not cut off by the default
|
||||
// top-5 limit when the database already holds other keys. (antd Segmented
|
||||
// renders label-wrapped radios, not options.)
|
||||
await card.locator(".ant-segmented-item").filter({ hasText: /^50$/ }).click();
|
||||
return card;
|
||||
}
|
||||
|
||||
test.describe("Usage page", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test("Top Virtual Keys lists a key that served traffic, toggles views, and opens key info", async ({
|
||||
page,
|
||||
request,
|
||||
}) => {
|
||||
const alias = `e2e-usage-key-${Date.now()}`;
|
||||
const { key, token } = await createVirtualKey(request, {
|
||||
key_alias: alias,
|
||||
});
|
||||
|
||||
const requestId = await sendChatCompletion(request, {
|
||||
model: CHAT_MODEL_A,
|
||||
prompt: `usage ping for ${alias}`,
|
||||
apiKey: key,
|
||||
});
|
||||
await waitForSpendLog(request, requestId);
|
||||
// Must land in the aggregate before the page mounts — it fetches once.
|
||||
await waitForKeyInDailyActivity(request, token);
|
||||
|
||||
const card = await openUsage(page);
|
||||
|
||||
// Table view (the default): the key is listed by its alias.
|
||||
const row = card.locator("tbody tr").filter({ hasText: alias });
|
||||
await expect(row, `${alias} missing from Top Virtual Keys`).toHaveCount(1, {
|
||||
timeout: 30_000,
|
||||
});
|
||||
|
||||
// Chart view swaps the table out for the bar chart, and back.
|
||||
await card.getByText("Chart View", { exact: true }).click();
|
||||
await expect(card.locator("tbody tr")).toHaveCount(0, { timeout: 10_000 });
|
||||
await card.getByText("Table View", { exact: true }).click();
|
||||
await expect(row).toHaveCount(1, { timeout: 10_000 });
|
||||
|
||||
// Clicking the Key ID cell fetches key info and opens the detail panel.
|
||||
// Assert on the panel's own controls, not on the alias: the alias is
|
||||
// already in the row behind the modal, so a text match on it would pass
|
||||
// even if the panel never opened.
|
||||
await row.locator("td").first().click();
|
||||
const keyInfo = page.getByRole("tab", { name: "Overview", exact: true });
|
||||
await expect(keyInfo, "key info panel did not open").toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
await expect(page.getByRole("tab", { name: "Settings", exact: true })).toBeVisible();
|
||||
await expect(page.getByText("Back to Keys", { exact: false })).toBeVisible();
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue