test(e2e/ui): cover team-scoped model visibility, re-editing litellm params, and model health checks (#40039)

* test(e2e/ui): cover team-scoped model visibility, re-editing litellm params, and model health checks

Three Models and Endpoints flows had no end-to-end coverage, and all three
keep coming back as bug reports.

modelsByTeam walks an internal user through the Current team control and
asserts the table lists exactly what each team grants. It creates one
deployment that belongs to no team, proves that deployment is visible under
Personal, then proves it is absent under both seeded teams, so an empty
table cannot pass the same assertions.

editLitellmParams adds a temperature and a custom pair to a deployment,
saves, then re-edits the temperature and drops the custom pair. It checks
both update request bodies, polls the stored deployment until the new
temperature is there, reloads the page to confirm the second save is what
renders, and sends one chat completion to prove the deployment still serves.

modelHealthStatus runs the health check on a reachable deployment and on one
pointed at a dead port, asserts the healthy and unhealthy cells and the two
detail dialogs, and reloads to confirm both statuses are stored.

Every deployment these specs create carries a unique name and is deleted in
afterEach, including on the failure path.

* test(e2e/ui): find health rows across every page of the health table

The health table pages server-side at 50 rows with no search box, so on a
proxy carrying more deployments than that the two deployments the spec
creates can land on a later page and the lookup finds nothing.

Row lookups now walk the pages, using the table's own page indicator to
know when to advance and when to wrap back to the first page.

* test(e2e/ui): build the created deployment ids without mutating the array

* test(ui): scope model deployments to Playwright fixtures
This commit is contained in:
yuneng-jiang 2026-09-08 22:49:40 -07:00 committed by GitHub
parent ee7c7e14f3
commit 0721163cac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 693 additions and 0 deletions

View file

@ -0,0 +1,196 @@
import {
test as base,
expect,
type Locator,
type Page as PlaywrightPage,
} from "@playwright/test";
import {
E2E_TEAM_CRUD_ALIAS,
E2E_TEAM_ORG_ALIAS,
INTERNAL_USER_STORAGE_PATH,
} from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
import { readBack } from "../../helpers/roundTrip";
import { CHAT_MODEL_A, CHAT_MODEL_B, masterKey } from "../../helpers/traffic";
const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`;
const CURRENT_TEAM_VIEW = "Current Team Models";
const ALL_MODELS_VIEW = "All Available Models";
const PERSONAL_TEAM = "Personal";
const teamSelector = (page: PlaywrightPage): Locator =>
page.getByRole("combobox", { name: "Current team", exact: true });
const viewSelector = (page: PlaywrightPage): Locator =>
page.getByRole("combobox", { name: "View", exact: true });
async function chooseOption(
page: PlaywrightPage,
selector: Locator,
optionName: string,
): Promise<void> {
await selector.click();
const option = page.getByRole("option", { name: optionName, exact: true });
await expect(option, `option ${optionName} is offered`).toBeVisible({
timeout: 10_000,
});
await option.click();
await expect(
selector,
`${optionName} is the selection the control now reports`,
).toContainText(optionName, {
timeout: 10_000,
});
}
async function deleteDeployment(
page: PlaywrightPage,
id: string,
): Promise<void> {
const post = () =>
page.request.post("/model/delete", {
headers: { Authorization: `Bearer ${masterKey()}` },
data: { id },
});
const deleted = await post().catch(() => post());
expect(
deleted.ok(),
`cleanup: /model/delete ${id} returned ${deleted.status()}`,
).toBe(true);
}
function modelRow(page: PlaywrightPage, modelName: string): Locator {
return page.getByRole("row").filter({ hasText: modelName });
}
async function isRegistered(
page: PlaywrightPage,
modelName: string,
): Promise<boolean> {
const body = await readBack<{ data: { model_name?: string }[] }>(
page,
"/v2/model/info",
);
return body.data.some((row) => row.model_name === modelName);
}
const uniqueSuffix = (): string =>
`${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const test = base.extend<{ ungrantedModelName: string }>({
ungrantedModelName: async ({ page }, use) => {
const ungrantedModelName = `e2e-ungranted-${uniqueSuffix()}`;
const created = await page.request.post("/model/new", {
headers: { Authorization: `Bearer ${masterKey()}` },
data: {
model_name: ungrantedModelName,
litellm_params: {
model: `openai/${ungrantedModelName}`,
api_base: MOCK_LLM_BASE,
api_key: "fake-key",
},
model_info: {},
},
});
expect(
created.ok(),
`/model/new failed: ${created.status()} ${await created.text()}`,
).toBe(true);
const ungrantedModelId = (await created.json()).model_info?.id;
expect(ungrantedModelId, "model id from /model/new").toBeTruthy();
try {
await expect
.poll(async () => await isRegistered(page, ungrantedModelName), {
message: `deployment ${ungrantedModelName} never appeared in /v2/model/info after create`,
timeout: 60_000,
})
.toBe(true);
await use(ungrantedModelName);
} finally {
await deleteDeployment(page, ungrantedModelId);
}
},
});
test.describe("Models and Endpoints for an internal user", () => {
test.use({ storageState: INTERNAL_USER_STORAGE_PATH });
test("shows an internal user exactly the models of the team they select", async ({
page,
ungrantedModelName,
}) => {
await navigateToPage(page, Page.Models);
await expect(
page.getByRole("tab", { name: "Your Models" }),
"an internal user lands on their own models tab, not an admin-only view",
).toBeVisible({ timeout: 15_000 });
await expect(
viewSelector(page),
"the models table opens scoped to the selected team",
).toContainText(CURRENT_TEAM_VIEW, { timeout: 15_000 });
await expect(
modelRow(page, ungrantedModelName),
`the personal view lists ${ungrantedModelName}, so it is on the proxy and reachable from this page`,
).toHaveCount(1, { timeout: 30_000 });
await chooseOption(page, teamSelector(page), E2E_TEAM_CRUD_ALIAS);
await expect(
modelRow(page, CHAT_MODEL_A),
`${E2E_TEAM_CRUD_ALIAS} lists ${CHAT_MODEL_A}`,
).toHaveCount(1, {
timeout: 15_000,
});
await expect(
modelRow(page, CHAT_MODEL_B),
`${E2E_TEAM_CRUD_ALIAS} lists ${CHAT_MODEL_B}`,
).toHaveCount(1, {
timeout: 15_000,
});
await expect(
modelRow(page, ungrantedModelName),
`${ungrantedModelName} is on the proxy but not granted to ${E2E_TEAM_CRUD_ALIAS}, so it must not be listed`,
).toHaveCount(0);
await chooseOption(page, teamSelector(page), E2E_TEAM_ORG_ALIAS);
await expect(
modelRow(page, CHAT_MODEL_A),
`${E2E_TEAM_ORG_ALIAS} lists ${CHAT_MODEL_A}`,
).toHaveCount(1, {
timeout: 15_000,
});
await expect(
page.getByTestId("pagination-range"),
`${E2E_TEAM_ORG_ALIAS} lists the one model it grants and nothing else`,
).toHaveText("Showing 1-1 of 1", { timeout: 15_000 });
await expect(
modelRow(page, CHAT_MODEL_B),
`${CHAT_MODEL_B} belongs to another team and must not leak into ${E2E_TEAM_ORG_ALIAS}`,
).toHaveCount(0);
await expect(
modelRow(page, ungrantedModelName),
`${ungrantedModelName} is granted to no team and must not leak into ${E2E_TEAM_ORG_ALIAS}`,
).toHaveCount(0);
await chooseOption(page, viewSelector(page), ALL_MODELS_VIEW);
await expect(
modelRow(page, CHAT_MODEL_A),
`switching to ${ALL_MODELS_VIEW} leaves the table populated rather than blanking it`,
).toHaveCount(1, { timeout: 15_000 });
await page.reload();
await expect(
teamSelector(page),
"the team selection is not persisted across a reload, so the table returns to the personal view",
).toContainText(PERSONAL_TEAM, { timeout: 15_000 });
await expect(
viewSelector(page),
"the view selection is not persisted across a reload either",
).toContainText(CURRENT_TEAM_VIEW, { timeout: 15_000 });
await expect(
modelRow(page, ungrantedModelName),
"the personal view still renders models after a reload rather than coming back empty",
).toHaveCount(1, { timeout: 30_000 });
});
});

View file

@ -0,0 +1,252 @@
import {
test as base,
expect,
type Page as PlaywrightPage,
} from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
import { captureRequestBody, readBack } from "../../helpers/roundTrip";
import { masterKey, sendChatCompletion } from "../../helpers/traffic";
const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`;
const CUSTOM_PARAM = "extra_headers";
const CUSTOM_PARAM_VALUE = { "X-E2E-Edit-Probe": "one" };
type StoredParams = Record<string, unknown>;
async function readStoredParams(
page: PlaywrightPage,
modelId: string,
): Promise<StoredParams> {
const body = await readBack<{ data: { litellm_params: StoredParams }[] }>(
page,
`/model/info?litellm_model_id=${modelId}`,
);
return body.data[0]?.litellm_params ?? {};
}
function paramsEditor(page: PlaywrightPage) {
return page.getByPlaceholder('"rpm": 100');
}
async function editParams(
page: PlaywrightPage,
mutate: (params: StoredParams) => StoredParams,
): Promise<void> {
await page.getByRole("button", { name: "Edit Settings" }).click();
const editor = paramsEditor(page);
await expect(
editor,
"the LiteLLM Params editor is reachable on every visit to the edit form",
).toBeVisible({
timeout: 15_000,
});
const shown = JSON.parse(await editor.inputValue()) as StoredParams;
await editor.fill(JSON.stringify(mutate(shown), null, 2));
}
async function deleteDeployment(
page: PlaywrightPage,
id: string,
): Promise<void> {
const post = () =>
page.request.post("/model/delete", {
headers: { Authorization: `Bearer ${masterKey()}` },
data: { id },
});
const deleted = await post().catch(() => post());
expect(
deleted.ok(),
`cleanup: /model/delete ${id} returned ${deleted.status()}`,
).toBe(true);
}
const uniqueSuffix = (): string =>
`${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const test = base.extend<{
deployment: { readonly modelName: string; readonly createdModelId: string };
}>({
deployment: async ({ page, request }, use) => {
const modelName = `e2e-edit-params-${uniqueSuffix()}`;
const created = await page.request.post("/model/new", {
headers: { Authorization: `Bearer ${masterKey()}` },
data: {
model_name: modelName,
litellm_params: {
model: `openai/${modelName}`,
api_base: MOCK_LLM_BASE,
api_key: "fake-key",
},
model_info: {},
},
});
expect(
created.ok(),
`/model/new failed: ${created.status()} ${await created.text()}`,
).toBe(true);
const createdModelId = (await created.json()).model_info?.id;
expect(createdModelId, "model id from /model/new").toBeTruthy();
try {
await expect
.poll(
async () => {
try {
await sendChatCompletion(request, {
model: modelName,
prompt: `warmup ${modelName}`,
});
return true;
} catch {
return false;
}
},
{
message: `deployment ${modelName} never became routable after /model/new`,
timeout: 60_000,
},
)
.toBe(true);
await use({ modelName, createdModelId });
} finally {
await deleteDeployment(page, createdModelId);
}
},
});
test.describe("Edit LiteLLM Params on a deployment", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("params added on a deployment can be re-edited, and the deployment keeps serving", async ({
page,
request,
deployment: { modelName, createdModelId },
}) => {
await navigateToPage(page, Page.Models);
const modelIdCell = page.getByTestId(`model-id-${createdModelId}`);
await expect(
modelIdCell,
`the Models table lists ${modelName}`,
).toBeVisible({ timeout: 15_000 });
await modelIdCell.click();
await expect(page.getByText("Back to Models").first()).toBeVisible({
timeout: 15_000,
});
await editParams(page, (params) => ({
...params,
temperature: 0.2,
[CUSTOM_PARAM]: CUSTOM_PARAM_VALUE,
}));
const firstSave = await captureRequestBody(
page,
{ method: "PATCH", urlIncludes: `/model/${createdModelId}/update` },
async () => {
await page.getByRole("button", { name: "Save Changes" }).click();
},
);
expect(
firstSave.litellm_params?.temperature,
"the added temperature goes on the wire",
).toBe(0.2);
expect(
firstSave.litellm_params?.[CUSTOM_PARAM],
`the added ${CUSTOM_PARAM} goes on the wire`,
).toEqual(CUSTOM_PARAM_VALUE);
expect(
firstSave.litellm_params?.model,
"a params edit does not rewrite the upstream model",
).toBe(`openai/${modelName}`);
expect(
firstSave.litellm_params?.api_base,
"a params edit does not rewrite the api base",
).toBe(MOCK_LLM_BASE);
expect(
firstSave.litellm_params,
"the credential is never re-sent, so a masked placeholder cannot overwrite the stored key",
).not.toHaveProperty("api_key");
await expect
.poll(
async () => (await readStoredParams(page, createdModelId)).temperature,
{
message: "the added temperature never reached the stored deployment",
timeout: 20_000,
},
)
.toBe(0.2);
const afterFirstSave = await readStoredParams(page, createdModelId);
expect(
afterFirstSave[CUSTOM_PARAM],
`the added ${CUSTOM_PARAM} reached the stored deployment`,
).toEqual(CUSTOM_PARAM_VALUE);
expect(
afterFirstSave.model,
"the stored upstream model survived the edit",
).toBe(`openai/${modelName}`);
expect(
afterFirstSave.api_base,
"the stored api base survived the edit",
).toBe(MOCK_LLM_BASE);
await editParams(page, (params) => ({
...Object.fromEntries(
Object.entries(params).filter(([key]) => key !== CUSTOM_PARAM),
),
temperature: 0.7,
}));
const secondSave = await captureRequestBody(
page,
{ method: "PATCH", urlIncludes: `/model/${createdModelId}/update` },
async () => {
await page.getByRole("button", { name: "Save Changes" }).click();
},
);
expect(
secondSave.litellm_params?.temperature,
"a param set by an earlier save can be edited again",
).toBe(0.7);
expect(
secondSave.litellm_params,
`dropping ${CUSTOM_PARAM} from the editor drops it from the request the UI sends`,
).not.toHaveProperty(CUSTOM_PARAM);
expect(
secondSave.litellm_params?.model,
"a second params edit still leaves the upstream model alone",
).toBe(`openai/${modelName}`);
expect(
secondSave.litellm_params?.api_base,
"a second params edit still leaves the api base alone",
).toBe(MOCK_LLM_BASE);
expect(
secondSave.litellm_params,
"the credential is still never re-sent",
).not.toHaveProperty("api_key");
await expect
.poll(
async () => (await readStoredParams(page, createdModelId)).temperature,
{
message:
"the re-edited temperature never reached the stored deployment",
timeout: 20_000,
},
)
.toBe(0.7);
await page.reload();
await expect(
page
.getByRole("tabpanel", { name: "Overview" })
.getByText('"temperature": 0.7'),
"reopening the deployment renders the re-edited value, not the one from the first save",
).toBeVisible({ timeout: 20_000 });
await sendChatCompletion(request, {
model: modelName,
prompt: `still serving ${modelName}`,
});
});
});

View file

@ -0,0 +1,245 @@
import {
test as base,
expect,
type Locator,
type Page as PlaywrightPage,
} from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
import { readBack } from "../../helpers/roundTrip";
import { masterKey } from "../../helpers/traffic";
const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`;
const UNREACHABLE_BASE = "http://127.0.0.1:9/v1";
async function isRegistered(
page: PlaywrightPage,
modelName: string,
): Promise<boolean> {
const body = await readBack<{ data: { model_name?: string }[] }>(
page,
"/v2/model/info",
);
return body.data.some((row) => row.model_name === modelName);
}
function healthRow(page: PlaywrightPage, modelName: string): Locator {
return page.getByRole("row").filter({ hasText: modelName });
}
function pageOf(label: string): { current: number; total: number } {
const [current, total] = label
.replace("Page ", "")
.split(" of ")
.map((part) => Number(part.trim()));
return { current, total };
}
async function locateHealthRow(
page: PlaywrightPage,
modelName: string,
): Promise<Locator> {
const pageLabel = page.getByTestId("pagination-page");
await expect(
pageLabel,
"the health table reports which page it is showing",
).toBeVisible({ timeout: 20_000 });
const deadline = Date.now() + 60_000;
while (Date.now() < deadline) {
const row = healthRow(page, modelName);
const onThisPage = await row
.first()
.waitFor({ state: "visible", timeout: 3_000 })
.then(() => true)
.catch(() => false);
if (onThisPage) return row;
const { current, total } = pageOf(await pageLabel.innerText());
const goTo = current < total ? current + 1 : 1;
if (total === 1) continue;
await page
.getByRole("button", {
name: current < total ? "Go to next page" : "Go to first page",
})
.click();
await expect(pageLabel).toContainText(`Page ${goTo} of`, {
timeout: 15_000,
});
}
return healthRow(page, modelName);
}
async function openHealthTab(page: PlaywrightPage): Promise<void> {
await page.getByRole("tab", { name: "Health Status" }).click();
await expect(
page.getByRole("heading", { name: "Model Health Status" }),
).toBeVisible({ timeout: 15_000 });
}
async function expectStatus(
page: PlaywrightPage,
modelName: string,
status: string,
): Promise<void> {
const row = await locateHealthRow(page, modelName);
await expect(row, `${modelName} has one row in the health table`).toHaveCount(
1,
{ timeout: 20_000 },
);
await expect(
row.getByText(status, { exact: true }),
`the Health Status cell for ${modelName} reads ${status}`,
).toHaveCount(1, { timeout: 60_000 });
}
async function deleteDeployment(
page: PlaywrightPage,
id: string,
): Promise<void> {
const post = () =>
page.request.post("/model/delete", {
headers: { Authorization: `Bearer ${masterKey()}` },
data: { id },
});
const deleted = await post().catch(() => post());
expect(
deleted.ok(),
`cleanup: /model/delete ${id} returned ${deleted.status()}`,
).toBe(true);
}
const uniqueSuffix = (): string =>
`${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
async function withDeployment(
page: PlaywrightPage,
prefix: string,
apiBase: string,
use: (name: string) => Promise<void>,
): Promise<void> {
const name = `${prefix}-${uniqueSuffix()}`;
const created = await page.request.post("/model/new", {
headers: { Authorization: `Bearer ${masterKey()}` },
data: {
model_name: name,
litellm_params: {
model: `openai/${name}`,
api_base: apiBase,
api_key: "fake-key",
},
model_info: {},
},
});
expect(
created.ok(),
`/model/new for ${name} failed: ${created.status()} ${await created.text()}`,
).toBe(true);
const id = (await created.json()).model_info?.id;
expect(id, `model id from /model/new for ${name}`).toBeTruthy();
try {
await expect
.poll(() => isRegistered(page, name), {
message: `deployment ${name} never appeared in /v2/model/info after create`,
timeout: 60_000,
})
.toBe(true);
await use(name);
} finally {
await deleteDeployment(page, id);
}
}
const test = base.extend<{ reachableName: string; unreachableName: string }>({
reachableName: async ({ page }, use) => {
await withDeployment(page, "e2e-health-up", MOCK_LLM_BASE, use);
},
unreachableName: async ({ page }, use) => {
await withDeployment(page, "e2e-health-down", UNREACHABLE_BASE, use);
},
});
test.describe("Model health status", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("Run Health Check reports a reachable deployment healthy and an unreachable one unhealthy", async ({
page,
reachableName,
unreachableName,
}) => {
await navigateToPage(page, Page.Models);
await openHealthTab(page);
for (const name of [reachableName, unreachableName]) {
const row = await locateHealthRow(page, name);
await expect(row, `${name} has one row in the health table`).toHaveCount(
1,
{ timeout: 20_000 },
);
await row
.getByRole("button", { name: "Run Health Check", exact: true })
.click();
}
await expectStatus(page, reachableName, "healthy");
await expect(
healthRow(page, reachableName).getByText("unhealthy", { exact: true }),
"a reachable deployment is never reported unhealthy",
).toHaveCount(0);
await expectStatus(page, unreachableName, "unhealthy");
const successDetail = (
await locateHealthRow(page, reachableName)
).getByRole("button", {
name: "View response details",
});
await expect(
successDetail,
`${reachableName} offers its health check response for inspection`,
).toBeVisible({ timeout: 60_000 });
await successDetail.click();
const successDialog = page.getByRole("dialog");
await expect(
successDialog.getByRole("heading", {
name: `Health Check Response - ${reachableName}`,
}),
"the healthy deployment's detail opens its own response dialog",
).toBeVisible({ timeout: 10_000 });
await successDialog.getByRole("button", { name: "Close" }).last().click();
await expect(successDialog).toBeHidden({ timeout: 10_000 });
const errorDetail = (
await locateHealthRow(page, unreachableName)
).getByRole("button", {
name: "View full error details",
});
await expect(
errorDetail,
`${unreachableName} offers its health check error for inspection`,
).toBeVisible({ timeout: 60_000 });
await errorDetail.click();
const errorDialog = page.getByRole("dialog");
await expect(
errorDialog.getByRole("heading", {
name: `Health Check Error - ${unreachableName}`,
}),
"the unreachable deployment's detail opens its own error dialog",
).toBeVisible({ timeout: 10_000 });
await expect(
errorDialog,
"the error dialog carries the upstream connection failure, not a generic message",
).toContainText(/connection error/i, { timeout: 10_000 });
await expect(
errorDialog,
"the error dialog names the endpoint that could not be reached",
).toContainText(UNREACHABLE_BASE);
await errorDialog.getByRole("button", { name: "Close" }).last().click();
await expect(errorDialog).toBeHidden({ timeout: 10_000 });
await page.reload();
await openHealthTab(page);
await expectStatus(page, reachableName, "healthy");
await expectStatus(page, unreachableName, "unhealthy");
});
});