From 7e6dfaf93fcad8548609cd69019b5236d9261938 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Tue, 8 Sep 2026 14:10:36 -0700 Subject: [PATCH] test(ui): scope model deployments to Playwright fixtures --- .../tests/internal-user/modelsByTeam.spec.ts | 48 ++++---- .../modelsPage/editLitellmParams.spec.ts | 80 ++++++------- .../modelsPage/modelHealthStatus.spec.ts | 106 +++++++++--------- 3 files changed, 113 insertions(+), 121 deletions(-) diff --git a/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts b/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts index f97f03483b3..5e2c80b5845 100644 --- a/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts +++ b/tests/e2e/ui/tests/internal-user/modelsByTeam.spec.ts @@ -1,5 +1,5 @@ import { - test, + test as base, expect, type Locator, type Page as PlaywrightPage, @@ -77,18 +77,11 @@ async function isRegistered( const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -test.describe("Models and Endpoints for an internal user", () => { - test.use({ storageState: INTERNAL_USER_STORAGE_PATH }); - - const auth = () => ({ Authorization: `Bearer ${masterKey()}` }); - - let ungrantedModelId = ""; - let ungrantedModelName = ""; - - test.beforeEach(async ({ page }) => { - ungrantedModelName = `e2e-ungranted-${uniqueSuffix()}`; +const test = base.extend<{ ungrantedModelName: string }>({ + ungrantedModelName: async ({ page }, use) => { + const ungrantedModelName = `e2e-ungranted-${uniqueSuffix()}`; const created = await page.request.post("/model/new", { - headers: auth(), + headers: { Authorization: `Bearer ${masterKey()}` }, data: { model_name: ungrantedModelName, litellm_params: { @@ -103,26 +96,29 @@ test.describe("Models and Endpoints for an internal user", () => { created.ok(), `/model/new failed: ${created.status()} ${await created.text()}`, ).toBe(true); - ungrantedModelId = (await created.json()).model_info?.id; + const ungrantedModelId = (await created.json()).model_info?.id; expect(ungrantedModelId, "model id from /model/new").toBeTruthy(); - await expect - .poll(async () => await isRegistered(page, ungrantedModelName), { - message: `deployment ${ungrantedModelName} never appeared in /v2/model/info after create`, - timeout: 60_000, - }) - .toBe(true); - }); + try { + await expect + .poll(async () => await isRegistered(page, ungrantedModelName), { + message: `deployment ${ungrantedModelName} never appeared in /v2/model/info after create`, + timeout: 60_000, + }) + .toBe(true); + await use(ungrantedModelName); + } finally { + await deleteDeployment(page, ungrantedModelId); + } + }, +}); - test.afterEach(async ({ page }) => { - if (!ungrantedModelId) return; - const id = ungrantedModelId; - ungrantedModelId = ""; - await deleteDeployment(page, id); - }); +test.describe("Models and Endpoints for an internal user", () => { + test.use({ storageState: INTERNAL_USER_STORAGE_PATH }); test("shows an internal user exactly the models of the team they select", async ({ page, + ungrantedModelName, }) => { await navigateToPage(page, Page.Models); diff --git a/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts b/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts index 1c321127a8d..4480515ae59 100644 --- a/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/editLitellmParams.spec.ts @@ -1,4 +1,8 @@ -import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; +import { + test as base, + expect, + type Page as PlaywrightPage, +} from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage } from "../../helpers/navigation"; @@ -61,18 +65,13 @@ async function deleteDeployment( const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -test.describe("Edit LiteLLM Params on a deployment", () => { - test.use({ storageState: ADMIN_STORAGE_PATH }); - - const auth = () => ({ Authorization: `Bearer ${masterKey()}` }); - - let createdModelId = ""; - let modelName = ""; - - test.beforeEach(async ({ page, request }) => { - modelName = `e2e-edit-params-${uniqueSuffix()}`; +const test = base.extend<{ + deployment: { readonly modelName: string; readonly createdModelId: string }; +}>({ + deployment: async ({ page, request }, use) => { + const modelName = `e2e-edit-params-${uniqueSuffix()}`; const created = await page.request.post("/model/new", { - headers: auth(), + headers: { Authorization: `Bearer ${masterKey()}` }, data: { model_name: modelName, litellm_params: { @@ -87,40 +86,43 @@ test.describe("Edit LiteLLM Params on a deployment", () => { created.ok(), `/model/new failed: ${created.status()} ${await created.text()}`, ).toBe(true); - createdModelId = (await created.json()).model_info?.id; + const createdModelId = (await created.json()).model_info?.id; expect(createdModelId, "model id from /model/new").toBeTruthy(); - await expect - .poll( - async () => { - try { - await sendChatCompletion(request, { - model: modelName, - prompt: `warmup ${modelName}`, - }); - return true; - } catch { - return false; - } - }, - { - message: `deployment ${modelName} never became routable after /model/new`, - timeout: 60_000, - }, - ) - .toBe(true); - }); + try { + await expect + .poll( + async () => { + try { + await sendChatCompletion(request, { + model: modelName, + prompt: `warmup ${modelName}`, + }); + return true; + } catch { + return false; + } + }, + { + message: `deployment ${modelName} never became routable after /model/new`, + timeout: 60_000, + }, + ) + .toBe(true); + await use({ modelName, createdModelId }); + } finally { + await deleteDeployment(page, createdModelId); + } + }, +}); - test.afterEach(async ({ page }) => { - if (!createdModelId) return; - const id = createdModelId; - createdModelId = ""; - await deleteDeployment(page, id); - }); +test.describe("Edit LiteLLM Params on a deployment", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); test("params added on a deployment can be re-edited, and the deployment keeps serving", async ({ page, request, + deployment: { modelName, createdModelId }, }) => { await navigateToPage(page, Page.Models); const modelIdCell = page.getByTestId(`model-id-${createdModelId}`); diff --git a/tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts b/tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts index ffb9a2de06f..247cce1b85d 100644 --- a/tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/modelHealthStatus.spec.ts @@ -1,5 +1,5 @@ import { - test, + test as base, expect, type Locator, type Page as PlaywrightPage, @@ -113,66 +113,60 @@ async function deleteDeployment( const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +async function withDeployment( + page: PlaywrightPage, + prefix: string, + apiBase: string, + use: (name: string) => Promise, +): Promise { + const name = `${prefix}-${uniqueSuffix()}`; + const created = await page.request.post("/model/new", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { + model_name: name, + litellm_params: { + model: `openai/${name}`, + api_base: apiBase, + api_key: "fake-key", + }, + model_info: {}, + }, + }); + expect( + created.ok(), + `/model/new for ${name} failed: ${created.status()} ${await created.text()}`, + ).toBe(true); + const id = (await created.json()).model_info?.id; + expect(id, `model id from /model/new for ${name}`).toBeTruthy(); + try { + await expect + .poll(() => isRegistered(page, name), { + message: `deployment ${name} never appeared in /v2/model/info after create`, + timeout: 60_000, + }) + .toBe(true); + await use(name); + } finally { + await deleteDeployment(page, id); + } +} + +const test = base.extend<{ reachableName: string; unreachableName: string }>({ + reachableName: async ({ page }, use) => { + await withDeployment(page, "e2e-health-up", MOCK_LLM_BASE, use); + }, + unreachableName: async ({ page }, use) => { + await withDeployment(page, "e2e-health-down", UNREACHABLE_BASE, use); + }, +}); + test.describe("Model health status", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); - const auth = () => ({ Authorization: `Bearer ${masterKey()}` }); - - let reachableName = ""; - let unreachableName = ""; - let createdIds: string[] = []; - - test.beforeEach(async ({ page }) => { - const stamp = uniqueSuffix(); - reachableName = `e2e-health-up-${stamp}`; - unreachableName = `e2e-health-down-${stamp}`; - createdIds = []; - - for (const [name, apiBase] of [ - [reachableName, MOCK_LLM_BASE], - [unreachableName, UNREACHABLE_BASE], - ] as const) { - const created = await page.request.post("/model/new", { - headers: auth(), - data: { - model_name: name, - litellm_params: { - model: `openai/${name}`, - api_base: apiBase, - api_key: "fake-key", - }, - model_info: {}, - }, - }); - expect( - created.ok(), - `/model/new for ${name} failed: ${created.status()} ${await created.text()}`, - ).toBe(true); - const id = (await created.json()).model_info?.id; - expect(id, `model id from /model/new for ${name}`).toBeTruthy(); - createdIds = [...createdIds, id]; - } - - for (const name of [reachableName, unreachableName]) { - await expect - .poll(async () => await isRegistered(page, name), { - message: `deployment ${name} never appeared in /v2/model/info after create`, - timeout: 60_000, - }) - .toBe(true); - } - }); - - test.afterEach(async ({ page }) => { - const ids = createdIds; - createdIds = []; - for (const id of ids) { - await deleteDeployment(page, id); - } - }); - test("Run Health Check reports a reachable deployment healthy and an unreachable one unhealthy", async ({ page, + reachableName, + unreachableName, }) => { await navigateToPage(page, Page.Models); await openHealthTab(page);