From fdc259077ea48225403bc16c0dbb964f4298eda2 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Mon, 31 Aug 2026 15:05:54 -0700 Subject: [PATCH] test(e2e/ui): automate 8 manual QA checklist flows Adds Playwright coverage for the RC checklist items an audit marked automatable today: Playground to Logs hand-off, public Agent/MCP hub tabs, team models in the Playground dropdown via a team key, Add Model with a stored credential, internal user team key creation, a second admin account, team model deletion, and Presidio guardrail CRUD without a live sidecar. Seeds e2e-team-keygen with the /key/generate member permission so the internal user key flow avoids the team-list cache lag --- tests/e2e/ui/constants.ts | 3 + tests/e2e/ui/fixtures/seed.sql | 14 ++- tests/e2e/ui/helpers/traffic.ts | 28 ++++++ .../ui/tests/guardrails/guardrails.spec.ts | 83 ++++++++++++++++ .../tests/internal-user/internalUser.spec.ts | 55 +++++++++++ .../internalUserWithTeams.spec.ts | 16 +++- tests/e2e/ui/tests/logs/logs.spec.ts | 26 ++++- tests/e2e/ui/tests/modelHub/modelHub.spec.ts | 81 +++++++++++++++- .../e2e/ui/tests/modelsPage/addModel.spec.ts | 81 ++++++++++++++++ .../tests/modelsPage/deleteTeamModel.spec.ts | 70 ++++++++++++++ .../ui/tests/proxy-admin/secondAdmin.spec.ts | 94 +++++++++++++++++++ .../e2e/ui/tests/team-admin/teamAdmin.spec.ts | 88 +++++++++++++++++ 12 files changed, 631 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/ui/tests/guardrails/guardrails.spec.ts create mode 100644 tests/e2e/ui/tests/modelsPage/deleteTeamModel.spec.ts create mode 100644 tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts diff --git a/tests/e2e/ui/constants.ts b/tests/e2e/ui/constants.ts index 9d918736262..bb33c90ddf3 100644 --- a/tests/e2e/ui/constants.ts +++ b/tests/e2e/ui/constants.ts @@ -29,6 +29,7 @@ export const E2E_PROXY_ADMIN_USER_ID = "e2e-proxy-admin"; export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local"; export const E2E_INTERNAL_USER_ID = "e2e-internal-user"; export const E2E_INTERNAL_USER_EMAIL = "internal@test.local"; +export const E2E_TEAM_ADMIN_USER_ID = "e2e-team-admin"; // Key aliases for seeded test keys (match seed.sql) export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey"; @@ -46,3 +47,5 @@ export const E2E_TEAM_ORG_ID = "e2e-team-org"; export const E2E_TEAM_ORG_ALIAS = "E2E Team In Org"; export const E2E_TEAM_NO_ADMIN_ID = "e2e-team-no-admin"; export const E2E_TEAM_NO_ADMIN_ALIAS = "E2E Team No Admin"; +export const E2E_TEAM_KEYGEN_ID = "e2e-team-keygen"; +export const E2E_TEAM_KEYGEN_ALIAS = "E2E Team Keygen"; diff --git a/tests/e2e/ui/fixtures/seed.sql b/tests/e2e/ui/fixtures/seed.sql index a1218633cdb..e77b4a16b3d 100644 --- a/tests/e2e/ui/fixtures/seed.sql +++ b/tests/e2e/ui/fixtures/seed.sql @@ -29,7 +29,7 @@ INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", VALUES ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), + ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org","e2e-team-keygen"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), @@ -63,6 +63,17 @@ INSERT INTO "LiteLLM_TeamTable" ( '[{"role":"user","user_id":"e2e-invitable-user"}]'::jsonb, '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false); +INSERT INTO "LiteLLM_TeamTable" ( + "team_id", "team_alias", "organization_id", "admins", "members", + "members_with_roles", "metadata", "models", "spend", "model_spend", "model_max_budget", "blocked", + "team_member_permissions" +) VALUES + ('e2e-team-keygen', 'E2E Team Keygen', NULL, + '{}', '{"e2e-internal-user"}', + '[{"role":"user","user_id":"e2e-internal-user"}]'::jsonb, + '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false, + '{"/key/generate"}'); + -- 6. Team Memberships (only user_id, team_id, spend — no created_at/updated_at) INSERT INTO "LiteLLM_TeamMembership" ("user_id", "team_id", "spend") VALUES @@ -72,6 +83,7 @@ VALUES ('e2e-removable-member', 'e2e-team-crud', 0.0), ('e2e-team-admin', 'e2e-team-delete', 0.0), ('e2e-internal-user', 'e2e-team-org', 0.0), + ('e2e-internal-user', 'e2e-team-keygen', 0.0), ('e2e-invitable-user', 'e2e-team-no-admin', 0.0); -- 7. Verification Tokens (API Keys) diff --git a/tests/e2e/ui/helpers/traffic.ts b/tests/e2e/ui/helpers/traffic.ts index a2fc9463c94..ebd3c9a417f 100644 --- a/tests/e2e/ui/helpers/traffic.ts +++ b/tests/e2e/ui/helpers/traffic.ts @@ -84,6 +84,34 @@ export async function waitForSpendLog( throw new Error(`spend log for request ${requestId} never appeared (last /spend/logs status ${lastStatus})`); } +export async function waitForSpendLogByPrompt( + request: APIRequestContext, + prompt: string, + timeoutMs = 60_000, +): Promise { + const deadline = Date.now() + timeoutMs; + let lastStatus = 0; + while (Date.now() < deadline) { + const res = await request.get(`${rootPath()}/spend/logs`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + lastStatus = res.status(); + if (res.ok()) { + const rows: { request_id?: string; messages?: unknown; proxy_server_request?: unknown }[] = await res.json(); + const row = (Array.isArray(rows) ? rows : []).find( + (candidate) => + JSON.stringify(candidate.messages ?? "").includes(prompt) || + JSON.stringify(candidate.proxy_server_request ?? "").includes(prompt), + ); + if (row?.request_id) { + return row.request_id; + } + } + await new Promise((r) => setTimeout(r, 2_000)); + } + throw new Error(`no spend log row carrying prompt ${prompt} appeared (last /spend/logs status ${lastStatus})`); +} + const isoDay = (d: Date): string => d.toISOString().slice(0, 10); /** diff --git a/tests/e2e/ui/tests/guardrails/guardrails.spec.ts b/tests/e2e/ui/tests/guardrails/guardrails.spec.ts new file mode 100644 index 00000000000..a3ce6a73075 --- /dev/null +++ b/tests/e2e/ui/tests/guardrails/guardrails.spec.ts @@ -0,0 +1,83 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, E2E_TEAM_NO_ADMIN_ID } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; + +test.describe("Guardrails", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Create a Presidio guardrail, see it in team settings, and delete it", async ({ page }) => { + const guardrailName = `e2e-presidio-${Date.now()}`; + + await navigateToPage(page, Page.Guardrails); + await dismissFeedbackPopup(page); + + await page.getByRole("button", { name: /Add New Guardrail/i }).click(); + await page.getByRole("menuitem", { name: "Add Provider Guardrail" }).click(); + + const dialog = page.getByRole("dialog", { name: "Create guardrail" }); + await expect(dialog).toBeVisible({ timeout: 10_000 }); + + await dialog.getByLabel("Guardrail Name").fill(guardrailName); + + const providerSelect = dialog.getByRole("combobox", { name: "Guardrail Provider" }); + await providerSelect.click(); + await providerSelect.fill("Presidio"); + await page.getByRole("option", { name: "Presidio PII" }).click(); + + await dialog.getByLabel("Mode", { exact: true }).click(); + await page.keyboard.type("pre_call"); + await expect(page.getByRole("option", { name: "pre_call" })).toBeAttached({ timeout: 5_000 }); + await page.keyboard.press("Enter"); + await expect(dialog.locator('[data-slot="combobox-chip"]').filter({ hasText: "pre_call" })).toBeVisible({ + timeout: 5_000, + }); + await dialog.getByText("Create guardrail", { exact: true }).click(); + + await dialog.getByLabel("presidio_analyzer_api_base").fill("http://127.0.0.1:9999"); + await expect(dialog.getByLabel("presidio_analyzer_api_base")).toHaveValue("http://127.0.0.1:9999"); + await dialog.getByLabel("presidio_anonymizer_api_base").fill("http://127.0.0.1:9999"); + await expect(dialog.getByLabel("presidio_anonymizer_api_base")).toHaveValue("http://127.0.0.1:9999"); + + await dialog.getByRole("button", { name: "Next" }).click(); + await expect(dialog.getByText("Configure PII Protection")).toBeVisible({ timeout: 10_000 }); + await dialog.getByRole("button", { name: "Select All & Mask" }).click(); + + await dialog.getByRole("button", { name: "Create Guardrail" }).click(); + await expect(page.getByText("Guardrail created successfully").first()).toBeVisible({ timeout: 15_000 }); + + const row = page.locator("table tbody tr").filter({ hasText: guardrailName }); + await expect(row).toHaveCount(1, { timeout: 15_000 }); + + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + await clickTeamId(page, E2E_TEAM_NO_ADMIN_ID); + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + + const guardrailsSelect = page.getByRole("combobox", { name: "Select guardrails" }); + await expect(guardrailsSelect).toBeVisible({ timeout: 10_000 }); + await guardrailsSelect.click(); + await guardrailsSelect.fill(guardrailName); + await expect(page.getByRole("option", { name: guardrailName })).toBeVisible({ timeout: 10_000 }); + await page.keyboard.press("Escape"); + + await navigateToPage(page, Page.Guardrails); + await expect(row).toHaveCount(1, { timeout: 15_000 }); + await row.getByRole("button", { name: "Open guardrail actions" }).click(); + await page.getByRole("menuitem", { name: "Delete" }).click(); + + const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); + await expect(deleteModal).toBeVisible({ timeout: 5_000 }); + await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); + + await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ + timeout: 10_000, + }); + await expect(row).toHaveCount(0, { timeout: 15_000 }); + + await page.reload(); + await expect(page.getByRole("button", { name: /Add New Guardrail/i })).toBeVisible({ timeout: 20_000 }); + await expect(page.locator("table tbody tr").filter({ hasText: guardrailName })).toHaveCount(0); + }); +}); diff --git a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts index b8424b06115..6c893e469c6 100644 --- a/tests/e2e/ui/tests/internal-user/internalUser.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUser.spec.ts @@ -3,10 +3,13 @@ import { E2E_INTERNAL_USER_KEY_ALIAS, E2E_TEAM_CRUD_ALIAS, E2E_TEAM_CRUD_ID, + E2E_TEAM_KEYGEN_ALIAS, INTERNAL_USER_STORAGE_PATH, } from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage, clickTeamId } from "../../helpers/navigation"; +import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic"; +import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground"; test.describe("Internal User", () => { test.use({ storageState: INTERNAL_USER_STORAGE_PATH }); @@ -38,6 +41,58 @@ test.describe("Internal User", () => { await expect(page.getByRole("tab", { name: "Members" })).not.toBeVisible(); }); + test("Internal user creates a team key and uses it in the Playground", async ({ page, request }) => { + const suffix = Date.now(); + const auth = { Authorization: `Bearer ${masterKey()}` }; + + let apiKey = ""; + try { + await navigateToPage(page, Page.ApiKeys); + + await page.getByRole("button", { name: /Create New Key/i }).click(); + await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); + + await expect(page.getByRole("radio", { name: "You", exact: true })).toBeVisible({ timeout: 10_000 }); + await expect(page.getByRole("radio", { name: "Another User" })).toHaveCount(0); + + const keyName = `e2e-internal-team-key-${suffix}`; + await page.getByLabel(/Key Name/).fill(keyName); + + const teamSelect = page.getByTestId("team-dropdown").getByRole("combobox"); + await teamSelect.click(); + await page.keyboard.type(E2E_TEAM_KEYGEN_ALIAS); + await page.locator('[data-slot="combobox-content"]:visible').getByText(E2E_TEAM_KEYGEN_ALIAS).first().click(); + + await page.getByRole("combobox", { name: "Select models" }).click(); + await page.getByRole("option", { name: "All Team Models", exact: true }).click(); + await page.keyboard.press("Escape"); + + await page.getByRole("button", { name: "Create Key", exact: true }).click(); + + await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 }); + apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim(); + expect(apiKey).toMatch(/^sk-/); + await page.keyboard.press("Escape"); + + await openPlayground(page); + await keySourceSelect(page).click(); + await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 }); + + const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key")); + await expect(keyInput).toBeVisible({ timeout: 10_000 }); + await keyInput.fill(apiKey); + + await selectModel(page, CHAT_MODEL_A); + await sendMessage(page, `internal user team key ping ${keyName}`); + + await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 }); + } finally { + if (apiKey) { + await request.post("/key/delete", { headers: auth, data: { keys: [apiKey] } }); + } + } + }); + test("Virtual Keys page does not surface litellm-dashboard team keys", async ({ page }) => { await navigateToPage(page, Page.ApiKeys); diff --git a/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts b/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts index 68319154554..23e9ed78d9c 100644 --- a/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts +++ b/tests/e2e/ui/tests/internal-user/internalUserWithTeams.spec.ts @@ -1,12 +1,17 @@ import { test, expect } from "@playwright/test"; -import { INTERNAL_USER_STORAGE_PATH, E2E_TEAM_CRUD_ALIAS, E2E_TEAM_ORG_ALIAS } from "../../constants"; +import { + INTERNAL_USER_STORAGE_PATH, + E2E_TEAM_CRUD_ALIAS, + E2E_TEAM_KEYGEN_ALIAS, + E2E_TEAM_ORG_ALIAS, +} from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage } from "../../helpers/navigation"; /** * Differential partner to internalUserNoTeam.spec.ts: the seeded - * e2e-internal-user belongs to exactly two teams, so the Create Key dropdown - * must list both. Without this, the no-team spec's "zero options" assertion + * e2e-internal-user belongs to exactly three teams, so the Create Key dropdown + * must list all of them. Without this, the no-team spec's "zero options" assertion * would still pass against a bug that empties the dropdown for everyone. */ test.describe("Internal User with team memberships", () => { @@ -24,10 +29,11 @@ test.describe("Internal User with team memberships", () => { const dropdown = page.locator('[data-slot="combobox-content"]:visible').first(); await expect(dropdown).toBeVisible({ timeout: 5_000 }); - // Both seeded memberships render, and nothing else does — proving the + // All seeded memberships render, and nothing else does — proving the // dropdown is scoped to the user's teams rather than empty or unfiltered. await expect(dropdown.getByText(E2E_TEAM_CRUD_ALIAS, { exact: true })).toBeVisible({ timeout: 10_000 }); await expect(dropdown.getByText(E2E_TEAM_ORG_ALIAS, { exact: true })).toBeVisible(); - await expect(dropdown.getByRole("option")).toHaveCount(2); + await expect(dropdown.getByText(E2E_TEAM_KEYGEN_ALIAS, { exact: true })).toBeVisible(); + await expect(dropdown.getByRole("option")).toHaveCount(3); }); }); diff --git a/tests/e2e/ui/tests/logs/logs.spec.ts b/tests/e2e/ui/tests/logs/logs.spec.ts index 56a3d0f0109..46dfbf47478 100644 --- a/tests/e2e/ui/tests/logs/logs.spec.ts +++ b/tests/e2e/ui/tests/logs/logs.spec.ts @@ -2,7 +2,14 @@ import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwr import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; -import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic"; +import { + CHAT_MODEL_A, + MOCK_RESPONSE_TEXT, + sendChatCompletion, + waitForSpendLog, + waitForSpendLogByPrompt, +} from "../../helpers/traffic"; +import { openPlayground, selectModel, sendMessage } from "../../helpers/playground"; /** * Anchored to traffic this spec generates itself, with a unique prompt and end user per run, so it @@ -47,6 +54,23 @@ test.describe("Logs page", () => { permissions: ["clipboard-read", "clipboard-write"], }); + test("a chat sent from the Playground lands in Logs with its content", async ({ page, request }) => { + const prompt = `logs-playground-prompt-${uniqueSuffix()}`; + await openPlayground(page); + await selectModel(page, CHAT_MODEL_A); + await sendMessage(page, prompt); + await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 }); + + const requestId = await waitForSpendLogByPrompt(request, prompt); + + const row = await openLogsForRequest(page, requestId); + await row.click(); + const drawer = page.getByRole("dialog").first(); + await expect(drawer.getByText("Request & Response")).toBeVisible({ timeout: 20_000 }); + await expect(drawer.getByText(prompt, { exact: false }).first()).toBeVisible({ timeout: 20_000 }); + await expect(drawer.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 20_000 }); + }); + test("a served request expands to its request and response", async ({ page, request }) => { const prompt = `logs-detail-prompt-${uniqueSuffix()}`; const requestId = await sendChatCompletion(request, { diff --git a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts index 16ec94c1dc8..7fe3894d75d 100644 --- a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts +++ b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts @@ -1,7 +1,8 @@ -import { test, expect } from "@playwright/test"; +import { test, expect, type APIRequestContext } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; +import { masterKey } from "../../helpers/traffic"; test.describe("AI Hub (internal admin view)", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); @@ -77,4 +78,82 @@ test.describe("Public model hub (/ui/model_hub_table)", () => { // agents/MCP servers exist, so we don't assert on them in a fresh CI run. await expect(page.getByRole("tab", { name: "Model Hub" })).toBeVisible({ timeout: 10_000 }); }); + + test("Agent Hub and MCP Hub tabs render their public entries", async ({ page, request }) => { + const suffix = `${Date.now()}`; + const agentName = `e2e-public-agent-${suffix}`; + const mcpServerName = `e2e_public_mcp_${suffix}`; + const auth = { Authorization: `Bearer ${masterKey()}` }; + + const seedPublicEntries = async (api: APIRequestContext): Promise<{ agentId: string; serverId: string }> => { + const agentRes = await api.post("/v1/agents", { + headers: auth, + data: { + agent_name: agentName, + agent_card_params: { + name: agentName, + description: "E2E public agent", + version: "1.0.0", + url: "http://127.0.0.1:9999/", + capabilities: {}, + skills: [], + defaultInputModes: ["text"], + defaultOutputModes: ["text"], + }, + }, + }); + expect(agentRes.ok(), `agent create failed (${agentRes.status()}): ${await agentRes.text()}`).toBe(true); + const agentId = (await agentRes.json()).agent_id as string; + + const serverRes = await api.post("/v1/mcp/server", { + headers: auth, + data: { + server_name: mcpServerName, + url: "http://127.0.0.1:9999/mcp", + transport: "http", + description: "E2E public MCP server", + }, + }); + expect(serverRes.ok(), `mcp server create failed (${serverRes.status()}): ${await serverRes.text()}`).toBe(true); + const serverId = (await serverRes.json()).server_id as string; + + const agentPublicRes = await api.post("/v1/agents/make_public", { + headers: auth, + data: { agent_ids: [agentId] }, + }); + expect(agentPublicRes.ok(), `agents make_public failed: ${await agentPublicRes.text()}`).toBe(true); + const mcpPublicRes = await api.post("/v1/mcp/make_public", { + headers: auth, + data: { mcp_server_ids: [serverId] }, + }); + expect(mcpPublicRes.ok(), `mcp make_public failed: ${await mcpPublicRes.text()}`).toBe(true); + + return { agentId, serverId }; + }; + + const { agentId, serverId } = await seedPublicEntries(request); + try { + await page.goto(`/ui/model_hub_table?key=${masterKey()}`); + await dismissFeedbackPopup(page); + + const agentHubTab = page.getByRole("tab", { name: "Agent Hub" }); + await expect(agentHubTab).toBeVisible({ timeout: 15_000 }); + await agentHubTab.click(); + await expect(page.getByText("Available Agents")).toBeVisible({ timeout: 10_000 }); + await expect(page.getByRole("row").filter({ hasText: agentName })).toHaveCount(1, { timeout: 10_000 }); + await expect(page.getByText("E2E public agent").first()).toBeVisible(); + + const mcpHubTab = page.getByRole("tab", { name: "MCP Hub" }); + await expect(mcpHubTab).toBeVisible(); + await mcpHubTab.click(); + await expect(page.getByText("Available MCP Servers")).toBeVisible({ timeout: 10_000 }); + await expect(page.getByRole("row").filter({ hasText: mcpServerName })).toHaveCount(1, { timeout: 10_000 }); + await expect(page.getByText("E2E public MCP server").first()).toBeVisible(); + } finally { + await request.post("/v1/agents/make_public", { headers: auth, data: { agent_ids: [] } }); + await request.post("/v1/mcp/make_public", { headers: auth, data: { mcp_server_ids: [] } }); + await request.delete(`/v1/agents/${agentId}`, { headers: auth }); + await request.delete(`/v1/mcp/server/${serverId}`, { headers: auth }); + } + }); }); diff --git a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts index dad716b4c83..f8ca02b36d0 100644 --- a/tests/e2e/ui/tests/modelsPage/addModel.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/addModel.spec.ts @@ -212,6 +212,87 @@ test.describe("Add Model", () => { .toBe(true); }); + test("Add a model with a stored credential, pass Test Connect, and serve traffic", async ({ page, request }) => { + const masterKey = users[Role.ProxyAdmin].password; + const auth = { Authorization: `Bearer ${masterKey}` }; + const credentialName = `e2e-cred-reuse-${Date.now()}`; + const createCred = await page.request.post("/credentials", { + headers: auth, + data: { + credential_name: credentialName, + credential_values: { api_key: "fake-key", api_base: MOCK_LLM_BASE }, + credential_info: { custom_llm_provider: "openai" }, + }, + }); + expect(createCred.ok(), `POST /credentials failed (${createCred.status()}): ${await createCred.text()}`).toBe(true); + + try { + await navigateToPage(page, Page.Models); + await page.getByRole("tab", { name: "Add Model" }).click(); + + await selectProvider(page, "OpenAI-Compatible Endpoints (Together AI, etc.)"); + + const publicName = `e2e-cred-model-${Date.now()}`; + uiAddedModelName = publicName; + + await page.getByRole("combobox", { name: "Select models" }).click(); + await page.getByRole("option", { name: "Custom Model Name (Enter below)" }).click(); + await page.keyboard.press("Escape"); + await page.getByPlaceholder("Enter custom model name").fill(publicName); + + const credentialSelect = page.getByRole("combobox", { name: "Existing Credentials" }); + await credentialSelect.click(); + await credentialSelect.fill(credentialName); + await page.getByRole("option", { name: credentialName, exact: true }).click(); + + await expect(page.locator("#api_key")).toHaveCount(0); + await expect(page.locator("#api_base")).toHaveCount(0); + + await page.getByRole("button", { name: "Test Connect" }).click(); + await expect(page.getByText("Connection Test Results")).toBeVisible({ timeout: 10_000 }); + await expect(page.getByTestId("connection-success-msg")).toBeVisible({ timeout: 30_000 }); + + const resultsModal = page.getByRole("dialog", { name: "Connection Test Results" }); + await resultsModal.locator('[data-slot="dialog-footer"]').getByRole("button", { name: "Close" }).click(); + await expect(resultsModal).toBeHidden({ timeout: 5_000 }); + + const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => { + await page.getByRole("button", { name: "Add Model" }).last().click(); + }); + expect(created.litellm_params?.litellm_credential_name, "the picked credential goes on the wire").toBe( + credentialName, + ); + expect(created.litellm_params?.api_key, "no raw api key goes on the wire").toBeUndefined(); + + await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 }); + + await expect + .poll( + async () => { + try { + await sendChatCompletion(request, { model: publicName, prompt: `hello via ${credentialName}` }); + return true; + } catch { + return false; + } + }, + { + message: `model ${publicName} added with a stored credential never served a request`, + timeout: 30_000, + }, + ) + .toBe(true); + } finally { + const stored = uiAddedModelName ? await findDeploymentByName(page, uiAddedModelName) : undefined; + const id = stored?.model_info?.id; + if (id) { + await page.request.post("/model/delete", { headers: auth, data: { id } }); + uiAddedModelName = ""; + } + await page.request.delete(`/credentials/${credentialName}`, { headers: auth }); + } + }); + test("Test connection with bad credentials shows failure", async ({ page }) => { await navigateToPage(page, Page.Models); await page.getByRole("tab", { name: "Add Model" }).click(); diff --git a/tests/e2e/ui/tests/modelsPage/deleteTeamModel.spec.ts b/tests/e2e/ui/tests/modelsPage/deleteTeamModel.spec.ts new file mode 100644 index 00000000000..dca7d9f006b --- /dev/null +++ b/tests/e2e/ui/tests/modelsPage/deleteTeamModel.spec.ts @@ -0,0 +1,70 @@ +import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, E2E_TEAM_CRUD_ID } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; +import { readBack } from "../../helpers/roundTrip"; +import { masterKey } from "../../helpers/traffic"; + +async function findDeploymentByName(page: PlaywrightPage, modelName: string): Promise | undefined> { + const body = await readBack<{ data: Record[] }>(page, "/v2/model/info"); + return body.data.find((row) => row.model_name === modelName); +} + +test.describe("Delete team model", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Delete a team-scoped model and verify it leaves the team's model list", async ({ page }) => { + const modelName = `e2e-team-model-delete-${Date.now()}`; + const createResponse = await page.request.post("/model/new", { + headers: { Authorization: `Bearer ${masterKey()}` }, + data: { + model_name: modelName, + litellm_params: { + model: "openai/fake-gpt-4", + api_base: `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`, + api_key: "fake-key", + }, + model_info: { team_id: E2E_TEAM_CRUD_ID }, + }, + }); + expect(createResponse.ok(), `/model/new failed: ${createResponse.status()} ${await createResponse.text()}`).toBe( + true, + ); + + await expect + .poll(async () => (await findDeploymentByName(page, modelName)) !== undefined, { + message: `deployment ${modelName} never appeared in /v2/model/info after create`, + timeout: 30_000, + }) + .toBe(true); + + await navigateToPage(page, Page.Models); + await page.getByPlaceholder("Search model names").fill(modelName); + + const row = page.locator("table tbody tr").filter({ hasText: modelName }); + await expect(row).toHaveCount(1, { timeout: 15_000 }); + await expect(row.getByText(E2E_TEAM_CRUD_ID)).toBeVisible({ timeout: 10_000 }); + + await row.getByRole("button", { name: "Delete model" }).click(); + + const modal = page.getByRole("dialog", { name: "Delete Model" }); + await expect(modal).toBeVisible({ timeout: 5_000 }); + await expect(modal.getByText(modelName).first()).toBeVisible(); + await modal.getByRole("button", { name: "Delete", exact: true }).click(); + + await expect(page.getByText("Model deleted successfully").first()).toBeVisible({ timeout: 10_000 }); + await expect(row).toHaveCount(0, { timeout: 15_000 }); + + await expect + .poll(async () => await findDeploymentByName(page, modelName), { + message: `deployment ${modelName} still readable from /v2/model/info after delete`, + timeout: 15_000, + }) + .toBeUndefined(); + + await page.reload(); + await page.getByPlaceholder("Search model names").fill(modelName); + await expect(page.getByText("No models found").first()).toBeVisible({ timeout: 15_000 }); + await expect(page.locator("table tbody tr").filter({ hasText: modelName })).toHaveCount(0); + }); +}); diff --git a/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts new file mode 100644 index 00000000000..2dd30060d2d --- /dev/null +++ b/tests/e2e/ui/tests/proxy-admin/secondAdmin.spec.ts @@ -0,0 +1,94 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic"; + +test.describe("Second proxy admin", () => { + test.use({ storageState: { cookies: [], origins: [] } }); + + test("an invited admin can log in, mint a key, and call a model with it", async ({ page, browser, request }) => { + const suffix = Date.now(); + const email = `second-admin-${suffix}@test.local`; + const password = "e2e-second-admin-password"; + const auth = { Authorization: `Bearer ${masterKey()}` }; + + const adminContext = await browser.newContext({ storageState: ADMIN_STORAGE_PATH }); + let userId = ""; + try { + const adminPage = await adminContext.newPage(); + await navigateToPage(adminPage, Page.Users); + await dismissFeedbackPopup(adminPage); + + await adminPage.getByRole("button", { name: "+ Invite User", exact: true }).click(); + const dialog = adminPage.getByRole("dialog", { name: "Invite User" }); + await expect(dialog).toBeVisible({ timeout: 5_000 }); + + await dialog.getByLabel("User Email").fill(email); + + await dialog.getByLabel(/Global Proxy Role/).click(); + await adminPage.getByRole("option", { name: /Admin \(All Permissions\)/ }).click(); + + const createdResponse = adminPage.waitForResponse( + (res) => res.url().includes("/user/new") && res.request().method() === "POST", + ); + await dialog.getByRole("button", { name: "Invite User" }).click(); + const createdBody = await (await createdResponse).json(); + userId = (createdBody.data?.user_id ?? createdBody.user_id) as string; + expect(userId, "created user id from /user/new").toBeTruthy(); + + await expect(adminPage.getByText("API user Created").first()).toBeVisible({ timeout: 10_000 }); + } finally { + await adminContext.close(); + } + + try { + const passwordRes = await request.post("/user/update", { + headers: auth, + data: { user_email: email, password }, + }); + expect(passwordRes.ok(), `setting password failed (${passwordRes.status()}): ${await passwordRes.text()}`).toBe( + true, + ); + + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill(email); + await page.getByPlaceholder("Enter your password").fill(password); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await expect(page.locator("a", { hasText: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); + await dismissFeedbackPopup(page); + + await navigateToPage(page, Page.ApiKeys); + await page.getByRole("button", { name: /Create New Key/i }).click(); + await expect(page.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); + + await page.getByLabel(/Key Name/).fill(`e2e-second-admin-key-${suffix}`); + + await page.getByRole("combobox", { name: "Select models" }).click(); + await page.getByRole("option", { name: "All Proxy Models", exact: true }).click(); + await page.keyboard.press("Escape"); + + await page.getByRole("button", { name: "Create Key", exact: true }).click(); + + await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 }); + const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim(); + expect(apiKey).toMatch(/^sk-/); + await page.keyboard.press("Escape"); + + const response = await page.request.post("/chat/completions", { + headers: { Authorization: `Bearer ${apiKey}` }, + data: { + model: CHAT_MODEL_A, + messages: [{ role: "user", content: `second admin ping ${suffix}` }], + }, + }); + expect(response.status()).toBe(200); + const body = await response.json(); + expect(body.choices?.[0]?.message?.content).toBe(MOCK_RESPONSE_TEXT); + } finally { + if (userId) { + await request.post("/user/delete", { headers: auth, data: { user_ids: [userId] } }); + } + } + }); +}); diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts index f93cca75347..c23fa2c994c 100644 --- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts +++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts @@ -1,6 +1,7 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; import { E2E_INTERNAL_USER_KEY_ALIAS, + E2E_TEAM_ADMIN_USER_ID, E2E_TEAM_CRUD_ALIAS, E2E_TEAM_CRUD_ID, TEAM_ADMIN_STORAGE_PATH, @@ -8,6 +9,8 @@ import { import { Page } from "../../fixtures/pages"; import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; import { captureRequestBody, readBack } from "../../helpers/roundTrip"; +import { CHAT_MODEL_A, masterKey } from "../../helpers/traffic"; +import { keySourceSelect, modelSelect, onlyVisible, openPlayground } from "../../helpers/playground"; /** * Every identifier a roster is addressable by. Which of user_id / user_email is populated depends on @@ -128,6 +131,91 @@ test.describe("Team Admin", () => { .not.toContain("e2e-removable-member"); }); + test("Team admin sees all team models in the Playground model dropdown", async ({ page, request }) => { + const suffix = Date.now(); + const teamModelName = `e2e-team-dropdown-model-${suffix}`; + const auth = { Authorization: `Bearer ${masterKey()}` }; + + const teamRes = await request.post("/team/new", { + headers: auth, + data: { + team_alias: `e2e-playground-team-${suffix}`, + models: [CHAT_MODEL_A], + members_with_roles: [{ role: "admin", user_id: E2E_TEAM_ADMIN_USER_ID }], + }, + }); + expect(teamRes.ok(), `team create failed (${teamRes.status()}): ${await teamRes.text()}`).toBe(true); + const teamId = (await teamRes.json()).team_id as string; + + let modelId = ""; + let teamKey = ""; + try { + const modelRes = await request.post("/model/new", { + headers: auth, + data: { + model_name: teamModelName, + litellm_params: { + model: "openai/fake-gpt-4", + api_base: `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`, + api_key: "fake-key", + }, + model_info: { team_id: teamId }, + }, + }); + expect(modelRes.ok(), `model create failed (${modelRes.status()}): ${await modelRes.text()}`).toBe(true); + modelId = (await modelRes.json()).model_info?.id as string; + + const keyRes = await request.post("/key/generate", { headers: auth, data: { team_id: teamId } }); + expect(keyRes.ok(), `key generate failed (${keyRes.status()}): ${await keyRes.text()}`).toBe(true); + teamKey = (await keyRes.json()).key as string; + + await expect + .poll( + async () => { + const res = await request.get("/model_group/info", { + headers: { Authorization: `Bearer ${teamKey}` }, + }); + if (!res.ok()) return false; + const body: { data?: { model_group?: string }[] } = await res.json(); + return (body.data ?? []).some((group) => group.model_group === teamModelName); + }, + { + message: `model group ${teamModelName} never became visible to the team key`, + timeout: 30_000, + }, + ) + .toBe(true); + + await openPlayground(page); + await keySourceSelect(page).click(); + await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 }); + + const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key")); + await expect(keyInput).toBeVisible({ timeout: 10_000 }); + await keyInput.fill(teamKey); + + const select = modelSelect(page); + await select.click(); + await select.fill(teamModelName); + await expect(onlyVisible(page.getByRole("option", { name: teamModelName }))).toBeVisible({ + timeout: 15_000, + }); + + await select.fill(CHAT_MODEL_A); + await expect(onlyVisible(page.getByRole("option", { name: CHAT_MODEL_A }))).toBeVisible({ + timeout: 15_000, + }); + } finally { + if (teamKey) { + await request.post("/key/delete", { headers: auth, data: { keys: [teamKey] } }); + } + if (modelId) { + await request.post("/model/delete", { headers: auth, data: { id: modelId } }); + } + await request.post("/team/delete", { headers: auth, data: { team_ids: [teamId] } }); + } + }); + test("Team admin can create a team key with All Team Models", async ({ page }) => { await navigateToPage(page, Page.ApiKeys); await dismissFeedbackPopup(page);