diff --git a/tests/e2e/ui/helpers/premium.ts b/tests/e2e/ui/helpers/premium.ts new file mode 100644 index 00000000000..28bc2e58bbc --- /dev/null +++ b/tests/e2e/ui/helpers/premium.ts @@ -0,0 +1,20 @@ +import * as fs from "fs"; +import { ADMIN_STORAGE_PATH } from "../constants"; + +/** + * Whether the proxy under test is licensed, read from the admin session JWT's `premium_user` + * claim. That is the same value the dashboard reads to enable premium-gated controls, so it + * describes the proxy Playwright is pointed at rather than the environment the runner happens + * to have, which are not the same machine when E2E_UI_BASE_URL points elsewhere. + */ +export function proxyIsPremium(): boolean { + const storage = JSON.parse(fs.readFileSync(ADMIN_STORAGE_PATH, "utf-8")) as { + cookies?: { name: string; value: string }[]; + }; + const token = storage.cookies?.find((cookie) => cookie.name === "token")?.value; + const payload = token?.split(".")[1]; + if (!payload) { + return false; + } + return JSON.parse(Buffer.from(payload, "base64url").toString("utf-8")).premium_user === true; +} diff --git a/tests/e2e/ui/helpers/traffic.ts b/tests/e2e/ui/helpers/traffic.ts index ebd3c9a417f..25eb671fd0e 100644 --- a/tests/e2e/ui/helpers/traffic.ts +++ b/tests/e2e/ui/helpers/traffic.ts @@ -4,12 +4,16 @@ import { APIRequestContext, expect } from "@playwright/test"; export const CHAT_MODEL_A = "fake-openai-gpt-4"; export const CHAT_MODEL_B = "fake-anthropic-claude"; +/** The deployment each of those models routes to, as spend logs and usage breakdowns name it. */ +export const DEPLOYMENT_MODEL_A = "openai/fake-gpt-4"; +export const DEPLOYMENT_MODEL_B = "openai/fake-claude"; + /** The only completion text fixtures/mock_llm_server/server.py ever returns. */ export const MOCK_RESPONSE_TEXT = "This is a mock response."; export const masterKey = (): string => process.env.LITELLM_MASTER_KEY || "sk-1234"; -const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? ""; +export const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? ""; interface ChatOptions { model: string; @@ -114,13 +118,54 @@ export async function waitForSpendLogByPrompt( const isoDay = (d: Date): string => d.toISOString().slice(0, 10); +interface DailyActivityKey { + metrics?: { api_requests?: number }; +} + +interface DailyActivityPage { + results?: { breakdown?: { api_keys?: Record } }[]; + metadata?: { total_pages?: number }; +} + +const requestsOnPage = (body: DailyActivityPage, keyToken: string): number => + (body.results ?? []).reduce((sum, day) => sum + (day.breakdown?.api_keys?.[keyToken]?.metrics?.api_requests ?? 0), 0); + +/** + * The route paginates its per-key breakdown. Reading only the first page finds a key while the + * database is small and stops finding it once a run has generated more keys than one page holds, + * which reads as "the rollup is not running" when the rollup is fine. + */ +async function keyRequestsInDailyActivity( + request: APIRequestContext, + query: string, + keyToken: string, + page = 1, + seen = 0, +): Promise { + const res = await request.get(`${rootPath()}/user/daily/activity?${query}&page=${page}`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + if (!res.ok()) { + return seen; + } + const body = (await res.json()) as DailyActivityPage; + const total = seen + requestsOnPage(body, keyToken); + return page >= (body.metadata?.total_pages ?? 1) + ? total + : keyRequestsInDailyActivity(request, query, keyToken, page + 1, total); +} + /** * The Usage page reads /user/daily/activity, a rollup written by a background job, and fetches it once * on mount. Navigating before the rollup lands leaves a stale render that never refreshes. + * + * The rollup lands request by request, so waiting only for the key to appear leaves a caller that + * sent several requests reading a partial count. Pass `minRequests` to wait for all of them. */ export async function waitForKeyInDailyActivity( request: APIRequestContext, keyToken: string, + minRequests = 1, timeoutMs = 120_000, ): Promise { const now = new Date(); @@ -129,25 +174,17 @@ export async function waitForKeyInDailyActivity( const query = `start_date=${isoDay(start)}&end_date=${isoDay(now)}`; const deadline = Date.now() + timeoutMs; - let lastStatus = 0; - while (Date.now() < deadline) { - const res = await request.get(`${rootPath()}/user/daily/activity?${query}`, { - headers: { Authorization: `Bearer ${masterKey()}` }, - }); - lastStatus = res.status(); - if (res.ok()) { - const body = await res.json(); - const seen = (body?.results ?? []).some( - (day: { breakdown?: { api_keys?: Record } }) => keyToken in (day.breakdown?.api_keys ?? {}), + for (;;) { + const seen = await keyRequestsInDailyActivity(request, query, keyToken); + if (seen >= minRequests) { + return; + } + if (Date.now() >= deadline) { + throw new Error( + `key ${keyToken} reached ${seen} of ${minRequests} requests in /user/daily/activity across every page; ` + + "the daily spend rollup may not be running", ); - if (seen) { - return; - } } await new Promise((r) => setTimeout(r, 3_000)); } - throw new Error( - `key ${keyToken} never appeared in /user/daily/activity (last status ${lastStatus}); ` + - "the daily spend rollup may not be running", - ); } diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts index 073d3c0b79c..de25ec1aac5 100644 --- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts @@ -5,6 +5,11 @@ import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; import { captureRequestBody, readBack } from "../../helpers/roundTrip"; import { sendChatCompletion } from "../../helpers/traffic"; +import { proxyIsPremium } from "../../helpers/premium"; + +/** Four probes 13s apart span 39s, one PROXY_CONFIG_RELOAD_INTERVAL_SECONDS (30s) plus margin. */ +const CREDENTIAL_PROBE_SUCCESSES = 4; +const CREDENTIAL_PROBE_SPACING_MS = 13_000; /** The mock LLM as the proxy reaches it: same host locally, a sidecar in the deployed stack. */ const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`; @@ -35,7 +40,10 @@ async function selectProvider(page: PlaywrightPage, providerName: string) { const providerDropdown = page.getByRole("combobox", { name: "Provider", exact: true }); await providerDropdown.click(); await providerDropdown.fill(providerName); - await page.getByRole("option").filter({ hasText: exactly(providerName) }).click(); + await page + .getByRole("option") + .filter({ hasText: exactly(providerName) }) + .click(); await expect(providerDropdown).toHaveValue(providerName); } @@ -78,6 +86,9 @@ test.describe("Add Model", () => { }); test("Edit team model TPM and RPM limits", async ({ page }) => { + // /model/new refuses a team-scoped deployment on an unlicensed proxy, so there this fails in + // setup on a product gate rather than on a regression in the edit it covers. + test.skip(!proxyIsPremium(), "proxy under test is unlicensed — team-scoped models are premium"); const masterKey = users[Role.ProxyAdmin].password; const modelName = `e2e-team-model-${Date.now()}`; @@ -226,8 +237,11 @@ test.describe("Add Model", () => { }); expect(createCred.ok(), `POST /credentials failed (${createCred.status()}): ${await createCred.text()}`).toBe(true); - // Multi-instance stacks propagate a new credential to the probe-serving instances on a periodic - // sync; consecutive successes guard against a load balancer alternating synced and stale replicas + // The proxy's periodic credential refresh prunes its in-memory list against a database snapshot + // it took before this credential landed, so a credential that resolves right after POST + // /credentials can stop resolving until the refresh after that. Successes spanning a whole + // PROXY_CONFIG_RELOAD_INTERVAL_SECONDS prove it survived a refresh, after which it stays. + // Resolution fails open onto the ambient key, so losing it reads as a confusing upstream 404. let consecutiveProbeSuccesses = 0; await expect .poll( @@ -249,11 +263,12 @@ test.describe("Add Model", () => { return consecutiveProbeSuccesses; }, { - message: `stored credential ${credentialName} never became usable for a connection test`, - timeout: 60_000, + message: `stored credential ${credentialName} never stayed usable across a config reload`, + intervals: [0, CREDENTIAL_PROBE_SPACING_MS], + timeout: 110_000, }, ) - .toBeGreaterThanOrEqual(3); + .toBeGreaterThanOrEqual(CREDENTIAL_PROBE_SUCCESSES); try { await navigateToPage(page, Page.Models); @@ -458,7 +473,7 @@ test.describe("Add Model", () => { await page.waitForLoadState("networkidle"); await page.getByPlaceholder("Search model names").fill("cohere"); - + // Clearer failure than timing out on a row assertion when the table is empty. await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, { timeout: 15_000, @@ -466,10 +481,7 @@ test.describe("Add Model", () => { // Pin to one row carrying both the name and the team, so the sibling test's // team-less cohere row can't satisfy it. - const teamCohereRow = page - .getByRole("row") - .filter({ hasText: "cohere/" }) - .filter({ hasText: E2E_TEAM_CRUD_ID }); + const teamCohereRow = page.getByRole("row").filter({ hasText: "cohere/" }).filter({ hasText: E2E_TEAM_CRUD_ID }); await expect(teamCohereRow).toHaveCount(1, { timeout: 15_000 }); } finally { await deleteTeamScopedCohereModels(); diff --git a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts index f5bee68f245..deb7ae70d07 100644 --- a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts @@ -1,15 +1,17 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH, - E2E_DELETE_KEY_ALIAS, E2E_REGENERATE_KEY_ALIAS, E2E_UPDATE_LIMITS_KEY_ALIAS, E2E_INTERNAL_USER_KEY_ALIAS, E2E_TEAM_CRUD_ALIAS, + E2E_TEAM_CRUD_ID, } from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; import { captureRequestBody, readBack } from "../../helpers/roundTrip"; +import { masterKey } from "../../helpers/traffic"; +import { proxyIsPremium } from "../../helpers/premium"; /** * Looks a key up by alias, undefined when none carries it. `return_full_object=true` is what makes @@ -23,6 +25,17 @@ async function findKeyByAlias(page: PlaywrightPage, alias: string): Promise row.key_alias === alias); } +/** A key this test owns, so deleting it costs the suite nothing on a retry or a second run. */ +async function createDeletableKey(page: PlaywrightPage): Promise { + const alias = `e2e-delete-key-${Date.now()}`; + const res = await page.request.post("/key/generate", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { key_alias: alias, team_id: E2E_TEAM_CRUD_ID }, + }); + expect(res.ok(), `POST /key/generate failed (${res.status()}): ${await res.text()}`).toBe(true); + return alias; +} + test.describe("Proxy Admin - Keys", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); @@ -67,6 +80,9 @@ test.describe("Proxy Admin - Keys", () => { }); test("Regenerate key", async ({ page }) => { + // The Regenerate Key button renders disabled when the proxy is unlicensed, so without one this + // fails on a product gate rather than on a regression. + test.skip(!proxyIsPremium(), "proxy under test is unlicensed — Regenerate Key is premium-gated"); await navigateToPage(page, Page.ApiKeys); await dismissFeedbackPopup(page); @@ -143,12 +159,16 @@ test.describe("Proxy Admin - Keys", () => { }); test("Delete key", async ({ page }) => { + // Deleting the seeded key leaves nothing for the next attempt, so the retries CI runs with are + // guaranteed to fail and the suite cannot run twice against one database. Bring our own. + const alias = await createDeletableKey(page); + await navigateToPage(page, Page.ApiKeys); await dismissFeedbackPopup(page); - const keyRow = page.getByRole("row").filter({ hasText: E2E_DELETE_KEY_ALIAS }); + const keyRow = page.getByRole("row").filter({ hasText: alias }); await expect(keyRow).toBeVisible({ timeout: 10_000 }); - await keyRow.getByRole("button", { name: E2E_DELETE_KEY_ALIAS }).click(); + await keyRow.getByRole("button", { name: alias }).click(); await expect(page.getByText("Back to Keys")).toBeVisible({ timeout: 10_000 }); @@ -157,7 +177,7 @@ test.describe("Proxy Admin - Keys", () => { const modal = page.getByRole("dialog", { name: "Delete Key" }); await expect(modal).toBeVisible({ timeout: 5_000 }); - await modal.locator("input").fill(E2E_DELETE_KEY_ALIAS); + await modal.locator("input").fill(alias); const deleteButton = modal.getByRole("button", { name: "Delete", exact: true }); await expect(deleteButton).toBeEnabled(); @@ -167,8 +187,8 @@ test.describe("Proxy Admin - Keys", () => { // The key is gone when the management API stops returning it, not when the toast says so. await expect - .poll(async () => await findKeyByAlias(page, E2E_DELETE_KEY_ALIAS), { - message: `key ${E2E_DELETE_KEY_ALIAS} still readable from /key/list after delete`, + .poll(async () => await findKeyByAlias(page, alias), { + message: `key ${alias} still readable from /key/list after delete`, timeout: 15_000, }) .toBeUndefined(); diff --git a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts index 7383b452162..303e4488e09 100644 --- a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts @@ -1,14 +1,9 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; -import { - ADMIN_STORAGE_PATH, - E2E_TEAM_CRUD_ID, - E2E_TEAM_DELETE_ALIAS, - E2E_TEAM_NO_ADMIN_ID, - E2E_TEAM_ORG_ID, -} from "../../constants"; +import { ADMIN_STORAGE_PATH, E2E_TEAM_CRUD_ID, E2E_TEAM_NO_ADMIN_ID, E2E_TEAM_ORG_ID } from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; import { readBack } from "../../helpers/roundTrip"; +import { masterKey } from "../../helpers/traffic"; /** GET /team/list returns a bare array of teams, each carrying team_alias/team_id. */ async function findTeamByAlias(page: PlaywrightPage, alias: string): Promise | undefined> { @@ -25,6 +20,17 @@ async function teamMemberEmails(page: PlaywrightPage, teamId: string): Promise member.user_email ?? "").filter(Boolean); } +/** A team this test owns, so deleting it costs the suite nothing on a retry or a second run. */ +async function createDeletableTeam(page: PlaywrightPage): Promise { + const alias = `e2e-delete-team-${Date.now()}`; + const res = await page.request.post("/team/new", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { team_alias: alias, models: ["fake-openai-gpt-4"] }, + }); + expect(res.ok(), `POST /team/new failed (${res.status()}): ${await res.text()}`).toBe(true); + return alias; +} + test.describe("Proxy Admin - Teams", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); @@ -121,10 +127,14 @@ test.describe("Proxy Admin - Teams", () => { }); test("Delete a team", async ({ page }) => { + // Deleting the seeded team leaves nothing for the next attempt, so the retries CI runs with are + // guaranteed to fail and the suite cannot run twice against one database. Bring our own. + const alias = await createDeletableTeam(page); + await navigateToPage(page, Page.Teams); await dismissFeedbackPopup(page); - const teamRow = page.locator("tr", { hasText: E2E_TEAM_DELETE_ALIAS }).first(); + const teamRow = page.locator("tr", { hasText: alias }).first(); await expect(teamRow).toBeVisible({ timeout: 10_000 }); // Actions live in a kebab menu: open it, then click "Delete team". await teamRow.locator('[data-testid^="team-actions-"]').click(); @@ -132,15 +142,15 @@ test.describe("Proxy Admin - Teams", () => { const modal = page.getByRole("dialog", { name: "Delete Team?" }); await expect(modal).toBeVisible({ timeout: 5_000 }); - await modal.locator("input").fill(E2E_TEAM_DELETE_ALIAS); + await modal.locator("input").fill(alias); await modal.getByRole("button", { name: /Force Delete|Delete/i }).click(); await expect(teamRow).not.toBeVisible({ timeout: 10_000 }); // A row vanishing is local state, which happens whether or not the delete landed. await expect - .poll(async () => await findTeamByAlias(page, E2E_TEAM_DELETE_ALIAS), { - message: `team ${E2E_TEAM_DELETE_ALIAS} still readable from /team/list after delete`, + .poll(async () => await findTeamByAlias(page, alias), { + message: `team ${alias} still readable from /team/list after delete`, timeout: 15_000, }) .toBeUndefined(); diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts index 26a6fa50b4b..f3c031f0172 100644 --- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts +++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts @@ -35,7 +35,44 @@ async function findKeyByAlias(page: PlaywrightPage, alias: string): Promise row.key_alias === alias); } +/** A member this test adds itself, so removing it costs the suite nothing on a retry or a re-run. */ +async function addRemovableMember(page: PlaywrightPage, registerForCleanup: string[]): Promise { + const userId = `e2e-removable-${Date.now()}`; + // Claimed before the call: /user/new can persist the user and still answer non-2xx, and the id is + // ours either way, so registering it up front is what no failure path can skip. + registerForCleanup.push(userId); + const created = await page.request.post("/user/new", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { user_id: userId, user_role: "internal_user", auto_create_key: false }, + }); + expect(created.ok(), `POST /user/new failed (${created.status()}): ${await created.text()}`).toBe(true); + + const added = await page.request.post("/team/member_add", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { team_id: E2E_TEAM_CRUD_ID, member: { user_id: userId, role: "user" } }, + }); + expect(added.ok(), `POST /team/member_add failed (${added.status()}): ${await added.text()}`).toBe(true); + return userId; +} + test.describe("Team Admin", () => { + const createdMembers: string[] = []; + + test.afterEach(async ({ page }) => { + // Runs on the failure path too, which a call at the end of the test body would not. Ids are + // claimed before the user is created, so the delete is attempted unconditionally and only its + // own 404 counts as never persisted; any other answer is a cleanup failure worth reporting + // rather than a reason to leave the user behind. + for (const userId of createdMembers.splice(0)) { + const deleted = await page.request.post("/user/delete", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { user_ids: [userId] }, + }); + const settled = deleted.ok() || deleted.status() === 404; + expect(settled, `POST /user/delete for ${userId} (${deleted.status()}): ${await deleted.text()}`).toBe(true); + } + }); + test.use({ storageState: TEAM_ADMIN_STORAGE_PATH }); test("Team admin can see all team keys including internal user keys", async ({ page }) => { @@ -95,6 +132,10 @@ test.describe("Team Admin", () => { }); test("Team admin can remove a member from their team", async ({ page }) => { + // Removing the seeded member leaves nothing for the next attempt, so the retries CI runs with + // are guaranteed to fail and the suite cannot run twice against one database. Bring our own. + const memberId = await addRemovableMember(page, createdMembers); + await navigateToPage(page, Page.Teams); await dismissFeedbackPopup(page); @@ -102,9 +143,9 @@ test.describe("Team Admin", () => { await page.getByRole("tab", { name: "Members" }).click(); - // Seeded members appear in the roster by user_id (members_with_roles has no - // email), so match the row on the user_id rather than the email. - const row = page.locator("tr", { hasText: "e2e-removable-member" }).first(); + // Members appear in the roster by user_id (members_with_roles has no email), so match + // the row on the user_id rather than the email. + const row = page.locator("tr", { hasText: memberId }).first(); await expect(row).toBeVisible({ timeout: 10_000 }); await row.getByTestId("delete-member").click(); @@ -117,7 +158,7 @@ test.describe("Team Admin", () => { // Removing the wrong member is exactly what a success toast hides, so pin both halves. expect(remove.team_id, "delete targets the team being viewed").toBe(E2E_TEAM_CRUD_ID); expect([remove.user_id, remove.user_email], "delete identifies the member whose row was clicked").toContain( - "e2e-removable-member", + memberId, ); await expect(page.getByText("Team member removed successfully").first()).toBeVisible({ timeout: 10_000 }); @@ -128,7 +169,7 @@ test.describe("Team Admin", () => { message: "removed member is still on the team", timeout: 15_000, }) - .not.toContain("e2e-removable-member"); + .not.toContain(memberId); }); test("Team admin sees all team models in the Playground model dropdown", async ({ page, request }) => { diff --git a/tests/e2e/ui/tests/usage/usagePage.spec.ts b/tests/e2e/ui/tests/usage/usagePage.spec.ts index f61ab018e1b..3d057cfa2c9 100644 --- a/tests/e2e/ui/tests/usage/usagePage.spec.ts +++ b/tests/e2e/ui/tests/usage/usagePage.spec.ts @@ -1,10 +1,11 @@ -import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwright/test"; +import { test, expect, type APIRequestContext, type Locator, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; import { - CHAT_MODEL_A, createVirtualKey, + masterKey, + rootPath, sendChatCompletion, waitForKeyInDailyActivity, waitForSpendLog, @@ -27,9 +28,105 @@ async function openUsage(page: PlaywrightPage): Promise { return card; } +/** The upstream fixtures/config.yml points its models at, so the mock server answers this too. */ +const MOCK_DEPLOYMENT = "openai/fake-gpt-4"; + +/** A deployment whose traffic costs real money, so the key that used it outranks the $0 crowd. */ +async function createPricedDeployment( + request: APIRequestContext, + label: string, + registerForCleanup: string[], +): Promise<{ modelName: string }> { + const modelName = `e2e-usage-priced-${label}`; + // Claimed before the call: /model/new can persist the deployment and still answer non-2xx, so a + // name recorded up front is the only registration no response shape can skip. + registerForCleanup.push(modelName); + const res = await request.post("/model/new", { + headers: { Authorization: `Bearer ${masterKey()}`, "Content-Type": "application/json" }, + data: { + model_name: modelName, + litellm_params: { + model: MOCK_DEPLOYMENT, + api_base: `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`, + api_key: "fake-key", + input_cost_per_token: 0.01, + output_cost_per_token: 0.01, + }, + }, + }); + expect(res.ok(), `POST /model/new failed (${res.status()}): ${await res.text()}`).toBe(true); + + // /model/new returns once the row is written, but the router only picks the deployment up on its + // next refresh, so sending traffic straight away can still get "no healthy deployments". A ping + // that fails writes no spend log, so retrying it costs the ranking this test asserts nothing. + await expect + .poll( + async () => { + const ping = await request.post(`${rootPath()}/v1/chat/completions`, { + headers: { Authorization: `Bearer ${masterKey()}`, "Content-Type": "application/json" }, + data: { model: modelName, messages: [{ role: "user", content: "readiness ping" }] }, + }); + return ping.ok(); + }, + { message: `deployment ${modelName} never became routable`, timeout: 60_000 }, + ) + .toBe(true); + + return { modelName }; +} + test.describe("Usage page", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); + const pricedDeployments: string[] = []; + + test.afterEach(async ({ request }) => { + // A deployment left behind keeps its custom pricing, so it goes on changing what later runs + // route and what they cost. Runs on the failure path too, which the test body would not. + // Resolved by name rather than by a returned id, so a create that persisted without answering + // 2xx is still cleaned up. /model/info serves the router, and /model/new answers 2xx even when + // its in-request router reload failed, so the search-backed listing is what covers a deployment + // that reached the database only. Absent from both means it never persisted. + const names = pricedDeployments.splice(0); + if (names.length === 0) return; + const auth = { Authorization: `Bearer ${masterKey()}`, "Content-Type": "application/json" }; + + type Lookup = + | { readonly listed: true; readonly id: string | undefined } + | { readonly listed: false; readonly status: number }; + + const idIn = async (path: string, name: string): Promise => { + const listed = await request.get(path, { headers: auth }); + if (!listed.ok()) return { listed: false, status: listed.status() }; + const deployments = ((await listed.json()).data ?? []) as { + model_name?: string; + model_info?: { id?: string }; + }[]; + return { listed: true, id: deployments.find((d) => d.model_name === name)?.model_info?.id }; + }; + + const remove = async (name: string, id: string) => { + const deleted = await request.post(`${rootPath()}/model/delete`, { headers: auth, data: { id } }); + expect(deleted.ok(), `POST /model/delete for ${name} (${deleted.status()})`).toBe(true); + }; + + for (const name of names) { + const fromRouter = await idIn(`${rootPath()}/model/info`, name); + if (fromRouter.listed && fromRouter.id !== undefined) { + await remove(name, fromRouter.id); + continue; + } + const search = encodeURIComponent(name); + const fromDb = await idIn(`${rootPath()}/v2/model/info?search=${search}`, name); + expect( + fromDb.listed, + `GET /v2/model/info?search=${search} (${fromDb.listed ? 200 : fromDb.status}), so ${name} could not be checked`, + ).toBe(true); + if (!fromDb.listed || fromDb.id === undefined) continue; + await remove(name, fromDb.id); + } + }); + test("Top Virtual Keys lists a key that served traffic, toggles views, and opens key info", async ({ page, request, @@ -39,8 +136,13 @@ test.describe("Usage page", () => { key_alias: alias, }); + // Top Virtual Keys ranks by spend, and every mock deployment costs $0, so once a run has more + // keys than the list shows, whether this one makes the cut is down to how ties happen to sort. + // Give it a priced deployment of its own so it earns its place. + const { modelName } = await createPricedDeployment(request, alias, pricedDeployments); + const requestId = await sendChatCompletion(request, { - model: CHAT_MODEL_A, + model: modelName, prompt: `usage ping for ${alias}`, apiKey: key, });