diff --git a/tests/e2e/ui/tests/proxy-admin/teamGuardrailRemoval.spec.ts b/tests/e2e/ui/tests/proxy-admin/teamGuardrailRemoval.spec.ts new file mode 100644 index 00000000000..f7930378b1e --- /dev/null +++ b/tests/e2e/ui/tests/proxy-admin/teamGuardrailRemoval.spec.ts @@ -0,0 +1,145 @@ +import { test, expect, type APIRequestContext, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; +import { proxyIsPremium } from "../../helpers/premium"; +import { readBack } from "../../helpers/roundTrip"; +import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic"; + +const auth = () => ({ Authorization: `Bearer ${masterKey()}` }); + +async function guardrailId(request: APIRequestContext, name: string): Promise { + const res = await request.get("/v2/guardrails/list", { headers: auth() }); + expect(res.ok(), `GET /v2/guardrails/list (${res.status()})`).toBe(true); + const rows = (await res.json()).guardrails as { guardrail_id: string; guardrail_name: string | null }[]; + return rows.find((row) => row.guardrail_name === name)?.guardrail_id; +} + +async function teamGuardrails(page: PlaywrightPage, teamId: string): Promise { + const body = await readBack<{ team_info: { metadata: { guardrails?: string[] } | null } }>( + page, + `/team/info?team_id=${encodeURIComponent(teamId)}`, + ); + return body.team_info.metadata?.guardrails ?? []; +} + +async function keywordPromptStatus(request: APIRequestContext, apiKey: string, keyword: string): Promise { + const res = await request.post("/v1/chat/completions", { + headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, + data: { model: CHAT_MODEL_A, messages: [{ role: "user", content: `please tell me about ${keyword}` }] }, + }); + return res.status(); +} + +test.describe("Proxy Admin - Team guardrail removal", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Clearing a team's only guardrail on the Settings tab lets blocked traffic through again", async ({ + page, + request, + }) => { + test.skip(!proxyIsPremium(), "proxy under test is unlicensed, so team guardrails are premium-gated"); + + const stamp = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`; + const guardrailName = `e2e-team-guardrail-${stamp}`; + const bannedKeyword = `e2eteamban${stamp}`; + const teamAlias = `e2e-guardrail-team-${stamp}`; + + let teamId = ""; + let teamKey = ""; + try { + const guardrailRes = await request.post("/guardrails", { + headers: auth(), + data: { + guardrail: { + guardrail_name: guardrailName, + litellm_params: { + guardrail: "litellm_content_filter", + mode: "pre_call", + default_on: false, + blocked_words: [{ keyword: bannedKeyword, action: "BLOCK" }], + }, + }, + }, + }); + expect( + guardrailRes.ok(), + `POST /guardrails failed (${guardrailRes.status()}): ${await guardrailRes.text()}`, + ).toBe(true); + + const teamRes = await request.post("/team/new", { + headers: auth(), + data: { team_alias: teamAlias, models: [CHAT_MODEL_A], metadata: { guardrails: [guardrailName] } }, + }); + expect(teamRes.ok(), `POST /team/new failed (${teamRes.status()}): ${await teamRes.text()}`).toBe(true); + teamId = (await teamRes.json()).team_id as string; + + const keyRes = await request.post("/key/generate", { headers: auth(), data: { team_id: teamId } }); + expect(keyRes.ok(), `POST /key/generate failed (${keyRes.status()}): ${await keyRes.text()}`).toBe(true); + teamKey = (await keyRes.json()).key as string; + + await expect + .poll(async () => keywordPromptStatus(request, teamKey, bannedKeyword), { + message: "the team's guardrail never started refusing the banned keyword", + timeout: 60_000, + }) + .toBe(400); + + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + await clickTeamId(page, teamId); + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + + const chip = page.locator('[data-slot="combobox-chip"]').filter({ hasText: guardrailName }); + await expect(chip).toBeVisible({ timeout: 10_000 }); + await chip.locator('[data-slot="combobox-chip-remove"]').click(); + await expect(chip).toHaveCount(0, { timeout: 10_000 }); + + await page.getByRole("button", { name: "Save Changes" }).click(); + + await expect + .poll(async () => teamGuardrails(page, teamId), { + message: "the team still carries a guardrail in /team/info after the save", + timeout: 20_000, + }) + .toEqual([]); + + await page.reload(); + await page.getByRole("tab", { name: "Settings" }).click(); + await page.getByRole("button", { name: "Edit Settings" }).click(); + await expect(page.getByRole("combobox", { name: "Select guardrails" })).toBeVisible({ timeout: 15_000 }); + await expect( + page.locator('[data-slot="combobox-chip"]').filter({ hasText: guardrailName }), + "the removed guardrail is gone from the Settings tab after a reload", + ).toHaveCount(0); + + await expect + .poll(async () => keywordPromptStatus(request, teamKey, bannedKeyword), { + message: "the team key is still refused for a keyword whose guardrail was removed", + timeout: 60_000, + }) + .toBe(200); + + const served = await request.post("/v1/chat/completions", { + headers: { Authorization: `Bearer ${teamKey}`, "Content-Type": "application/json" }, + data: { + model: CHAT_MODEL_A, + messages: [{ role: "user", content: `please tell me about ${bannedKeyword}` }], + }, + }); + expect((await served.json()).choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT); + } finally { + if (teamKey) { + await request.post("/key/delete", { headers: auth(), data: { keys: [teamKey] } }); + } + if (teamId) { + await request.post("/team/delete", { headers: auth(), data: { team_ids: [teamId] } }); + } + const id = await guardrailId(request, guardrailName); + if (id) { + await request.delete(`/guardrails/${id}`, { headers: auth() }); + } + } + }); +}); diff --git a/tests/e2e/ui/tests/proxy-admin/teamMemberEdit.spec.ts b/tests/e2e/ui/tests/proxy-admin/teamMemberEdit.spec.ts new file mode 100644 index 00000000000..c7325e78f75 --- /dev/null +++ b/tests/e2e/ui/tests/proxy-admin/teamMemberEdit.spec.ts @@ -0,0 +1,129 @@ +import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; +import { readBack } from "../../helpers/roundTrip"; +import { CHAT_MODEL_A, masterKey } from "../../helpers/traffic"; + +interface TeamInfoResponse { + team_info: { + models: string[]; + members_with_roles: { user_id?: string; role?: string }[]; + }; + team_memberships: { + user_id: string; + litellm_budget_table: { max_budget: number | null } | null; + }[]; +} + +const auth = () => ({ Authorization: `Bearer ${masterKey()}` }); + +async function teamInfo(page: PlaywrightPage, teamId: string): Promise { + return readBack(page, `/team/info?team_id=${encodeURIComponent(teamId)}`); +} + +function roleOf(info: TeamInfoResponse, userId: string): string | undefined { + return info.team_info.members_with_roles.find((member) => member.user_id === userId)?.role; +} + +function budgetOf(info: TeamInfoResponse, userId: string): number | null | undefined { + return info.team_memberships.find((membership) => membership.user_id === userId)?.litellm_budget_table?.max_budget; +} + +function otherMembers(info: TeamInfoResponse, userId: string): string[] { + return info.team_info.members_with_roles + .filter((member) => member.user_id !== userId) + .map((member) => `${member.user_id}:${member.role}`) + .sort(); +} + +test.describe("Proxy Admin - Team member edit", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + const createdTeams: string[] = []; + const createdUsers: string[] = []; + + test.afterEach(async ({ request }) => { + for (const teamId of createdTeams.splice(0)) { + await request.post("/team/delete", { headers: auth(), data: { team_ids: [teamId] } }); + } + for (const userId of createdUsers.splice(0)) { + await request.post("/user/delete", { headers: auth(), data: { user_ids: [userId] } }); + } + }); + + test("Editing a member's role and per-member budget persists and survives a reload", async ({ page, request }) => { + const stamp = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`; + const memberId = `e2e-member-edit-${stamp}`; + const teamAlias = `e2e-member-edit-team-${stamp}`; + + createdUsers.push(memberId); + const created = await request.post("/user/new", { + headers: auth(), + data: { user_id: memberId, user_role: "internal_user", auto_create_key: false }, + }); + expect(created.ok(), `POST /user/new failed (${created.status()}): ${await created.text()}`).toBe(true); + + const teamRes = await request.post("/team/new", { + headers: auth(), + data: { + team_alias: teamAlias, + models: [CHAT_MODEL_A], + members_with_roles: [{ user_id: memberId, role: "admin" }], + }, + }); + expect(teamRes.ok(), `POST /team/new failed (${teamRes.status()}): ${await teamRes.text()}`).toBe(true); + const teamId = (await teamRes.json()).team_id as string; + createdTeams.push(teamId); + + const before = await teamInfo(page, teamId); + expect(roleOf(before, memberId), "the member starts out as a team admin").toBe("admin"); + expect(budgetOf(before, memberId) ?? null, "the member starts out with no per-member budget").toBeNull(); + expect( + otherMembers(before, memberId).length, + "the team has another member for the edit to leave alone", + ).toBeGreaterThan(0); + + await navigateToPage(page, Page.Teams); + await dismissFeedbackPopup(page); + await clickTeamId(page, teamId); + await page.getByRole("tab", { name: "Members" }).click(); + + const memberRow = page.locator("tr", { hasText: memberId }).first(); + await expect(memberRow).toBeVisible({ timeout: 10_000 }); + await memberRow.getByTestId("edit-member").click(); + + const modal = page.getByRole("dialog", { name: "Edit Member" }); + await expect(modal).toBeVisible({ timeout: 10_000 }); + + await modal.getByLabel(/^Role/).click(); + await page.getByRole("option", { name: "User", exact: true }).click(); + await modal.getByLabel(/Team Member Budget \(USD\)/).fill("5"); + await modal.getByRole("button", { name: "Save Changes" }).click(); + + await expect(page.getByText("Team member updated successfully").first()).toBeVisible({ timeout: 10_000 }); + + await expect + .poll( + async () => { + const info = await teamInfo(page, teamId); + return [roleOf(info, memberId), budgetOf(info, memberId)]; + }, + { message: "the member's role and budget never landed in /team/info", timeout: 20_000 }, + ) + .toEqual(["user", 5]); + + await page.reload(); + await page.getByRole("tab", { name: "Members" }).click(); + const reloadedRow = page.locator("tr", { hasText: memberId }).first(); + await expect(reloadedRow).toBeVisible({ timeout: 15_000 }); + await expect(reloadedRow.getByText("user", { exact: true }), "role shown after a reload").toBeVisible(); + await expect(reloadedRow.getByText("$5.00"), "per-member budget shown after a reload").toBeVisible(); + + const after = await teamInfo(page, teamId); + expect(after.team_info.models, "model access untouched by a member edit").toEqual(before.team_info.models); + expect(otherMembers(after, memberId), "the rest of the roster untouched by a member edit").toEqual( + otherMembers(before, memberId), + ); + }); +}); diff --git a/tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts b/tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts new file mode 100644 index 00000000000..75fb3be9b64 --- /dev/null +++ b/tests/e2e/ui/tests/team-admin/memberPermissions.spec.ts @@ -0,0 +1,177 @@ +import { test, expect, type Browser, type BrowserContext, type Page as PlaywrightPage } from "@playwright/test"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation"; +import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic"; + +const PASSWORD = "E2e-Member-Perms-Pass-1!"; + +const auth = () => ({ Authorization: `Bearer ${masterKey()}` }); + +async function sessionKey(page: PlaywrightPage): Promise { + const cookie = (await page.context().cookies()).find((candidate) => candidate.name === "token"); + expect(cookie?.value, "logged-in session carries a token cookie").toBeTruthy(); + const payload = JSON.parse(Buffer.from(cookie!.value.split(".")[1], "base64url").toString("utf-8")) as { + key?: string; + }; + expect(payload.key, "session JWT carries the virtual key the dashboard calls with").toMatch(/^sk-/); + return payload.key!; +} + +async function signIn(browser: Browser, email: string): Promise { + const context = await browser.newContext({ storageState: { cookies: [], origins: [] } }); + const page = await context.newPage(); + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill(email); + await page.getByPlaceholder("Enter your password").fill(PASSWORD); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await expect(page.locator("a", { hasText: "Virtual Keys" })).toBeVisible({ timeout: 30_000 }); + await dismissFeedbackPopup(page); + return context; +} + +test.describe("Team Admin - Member permissions", () => { + test.use({ storageState: { cookies: [], origins: [] } }); + + test("Granting /key/generate lets a plain member create a team key that serves traffic", async ({ + browser, + request, + }) => { + const stamp = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`; + const adminId = `e2e-perm-admin-${stamp}`; + const memberId = `e2e-perm-member-${stamp}`; + const adminEmail = `${adminId}@test.local`; + const memberEmail = `${memberId}@test.local`; + const teamAlias = `e2e-perm-team-${stamp}`; + + const createUser = async (userId: string, email: string): Promise => { + const created = await request.post("/user/new", { + headers: auth(), + data: { user_id: userId, user_email: email, user_role: "internal_user", auto_create_key: false }, + }); + expect(created.ok(), `POST /user/new for ${userId} (${created.status()}): ${await created.text()}`).toBe(true); + const password = await request.post("/user/update", { + headers: auth(), + data: { user_id: userId, password: PASSWORD }, + }); + expect(password.ok(), `POST /user/update for ${userId} (${password.status()})`).toBe(true); + }; + + let teamId = ""; + const createdKeys: string[] = []; + const contexts: BrowserContext[] = []; + try { + await createUser(adminId, adminEmail); + await createUser(memberId, memberEmail); + + const teamRes = await request.post("/team/new", { + headers: auth(), + data: { + team_alias: teamAlias, + models: [CHAT_MODEL_A], + members_with_roles: [ + { user_id: adminId, role: "admin" }, + { user_id: memberId, role: "user" }, + ], + }, + }); + expect(teamRes.ok(), `POST /team/new failed (${teamRes.status()}): ${await teamRes.text()}`).toBe(true); + teamId = (await teamRes.json()).team_id as string; + + const memberContext = await signIn(browser, memberEmail); + contexts.push(memberContext); + const memberPage = memberContext.pages()[0]; + const memberSessionKey = await sessionKey(memberPage); + + const refused = await memberPage.request.post("/key/generate", { + headers: { Authorization: `Bearer ${memberSessionKey}`, "Content-Type": "application/json" }, + data: { team_id: teamId, key_alias: `e2e-perm-denied-${stamp}` }, + }); + expect(refused.status(), "a plain member cannot mint a team key before the grant").toBe(401); + expect(await refused.text()).toContain("/key/generate"); + + const adminContext = await signIn(browser, adminEmail); + contexts.push(adminContext); + const adminPage = adminContext.pages()[0]; + await navigateToPage(adminPage, Page.Teams); + await clickTeamId(adminPage, teamId); + await adminPage.getByRole("tab", { name: "Member Permissions" }).click(); + + for (const route of ["/key/generate", "/key/update"]) { + await adminPage.getByRole("row").filter({ hasText: route }).getByRole("checkbox").check(); + } + await adminPage.getByRole("button", { name: "Save Changes" }).click(); + await expect(adminPage.getByText("Permissions updated successfully").first()).toBeVisible({ timeout: 10_000 }); + + await expect + .poll( + async () => { + const res = await request.get(`/team/permissions_list?team_id=${encodeURIComponent(teamId)}`, { + headers: auth(), + }); + if (!res.ok()) return []; + return ((await res.json()).team_member_permissions ?? []) as string[]; + }, + { message: "the granted permissions never landed in /team/permissions_list", timeout: 20_000 }, + ) + .toEqual(expect.arrayContaining(["/key/generate", "/key/update"])); + + const keyAlias = `e2e-perm-key-${stamp}`; + await navigateToPage(memberPage, Page.ApiKeys); + await memberPage.getByRole("button", { name: /Create New Key/i }).click(); + await expect(memberPage.getByText("Key Ownership")).toBeVisible({ timeout: 10_000 }); + await memberPage.getByLabel(/Key Name/).fill(keyAlias); + + const teamSelect = memberPage.getByTestId("team-dropdown").getByRole("combobox"); + await teamSelect.click(); + await memberPage.keyboard.type(teamAlias); + await memberPage.getByRole("option", { name: teamAlias }).first().click(); + + await memberPage.getByRole("combobox", { name: "Select models" }).click(); + await memberPage.getByRole("option", { name: "All Team Models", exact: true }).click(); + await memberPage.keyboard.press("Escape"); + + await memberPage.getByRole("button", { name: "Create Key", exact: true }).click(); + const saveDialog = memberPage.getByRole("dialog", { name: "Save your Key" }); + await expect(saveDialog).toBeVisible({ timeout: 15_000 }); + const apiKey = (await saveDialog.locator("pre").innerText()).trim(); + expect(apiKey).toMatch(/^sk-/); + createdKeys.push(apiKey); + await memberPage.keyboard.press("Escape"); + + await expect + .poll( + async () => { + const res = await request.get( + `/key/list?team_id=${encodeURIComponent(teamId)}&return_full_object=true&size=100`, + { headers: auth() }, + ); + if (!res.ok()) return null; + const row = ((await res.json()).keys as Record[]).find( + (candidate) => candidate.key_alias === keyAlias, + ); + return row ? [row.user_id, row.team_id] : null; + }, + { message: `key ${keyAlias} never appeared on the team with the member as its owner`, timeout: 20_000 }, + ) + .toEqual([memberId, teamId]); + + const served = await request.post("/v1/chat/completions", { + headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, + data: { model: CHAT_MODEL_A, messages: [{ role: "user", content: `member key ping ${stamp}` }] }, + }); + expect(served.status(), "the delegated key is a real key the gateway serves").toBe(200); + expect((await served.json()).choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT); + } finally { + for (const context of contexts) { + await context.close(); + } + for (const key of createdKeys) { + await request.post("/key/delete", { headers: auth(), data: { keys: [key] } }); + } + if (teamId) { + await request.post("/team/delete", { headers: auth(), data: { team_ids: [teamId] } }); + } + await request.post("/user/delete", { headers: auth(), data: { user_ids: [adminId, memberId] } }); + } + }); +});