From b2b69d4f4d669a050b23007da07d373a236d6862 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:09:44 -0700 Subject: [PATCH] fix(e2e/ui): seed the playwright fixtures over the management API The ui suite has never gotten past globalSetup on stage. It logs in as five roles, and four of them are password users that only ever existed because run_e2e.sh and CircleCI piped fixtures/seed.sql into a database they had direct access to. The packaged e2e image has no such access, so on stage only the master-key admin could log in; the second role sat on /ui/login until waitForURL hit its 30s timeout and the whole run died before a single test. Confirmed against the stage gateway: POST /login as admin returns 303, as adminviewer@test.local returns 401, and /user/info 404s for every seeded id. Create the same budget, organization, users, teams and keys through the management endpoints in globalSetup instead, so the suite seeds whatever deployment it can reach rather than whatever database it can open. seed.sql is gone and the psql steps go with it; the API path is also the more faithful fixture, since seed.sql hand-wrote team rows no product code path produces (populated admins/members arrays, unhashed key tokens). Two behaviors the seeder has to work around, both verified live: /team/new auto-adds the caller as an admin, which would have made the proxy admin a member of the team the suite needs it to be a stranger to, so that membership is removed after each team; and /team/delete and /user/delete are all-or-nothing, so one unknown id in a batch deletes nothing, which means each id gets its own request for the cleanup to survive a partial state. Verified end to end against the stage gateway: seeding runs in ~6s, all five role logins return 303, the four teams come back with exactly their seeded members and no stray admin, all five keys resolve to the right owner and team, and re-running after deleting a seeded team and key restores both. --- .circleci/config.yml | 10 - tests/e2e/ui/constants.ts | 26 +- tests/e2e/ui/fixtures/seed.sql | 86 ------ tests/e2e/ui/fixtures/seed.ts | 272 ++++++++++++++++++ tests/e2e/ui/fixtures/users.ts | 21 +- tests/e2e/ui/globalSetup.ts | 7 + tests/e2e/ui/run_e2e.sh | 16 +- .../e2e/ui/tests/team-admin/teamAdmin.spec.ts | 3 +- 8 files changed, 320 insertions(+), 121 deletions(-) delete mode 100644 tests/e2e/ui/fixtures/seed.sql create mode 100644 tests/e2e/ui/fixtures/seed.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f01b6de4f3..caeb08a7498 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2776,11 +2776,6 @@ jobs: - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py @@ -2904,11 +2899,6 @@ jobs: - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - run: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py diff --git a/tests/e2e/ui/constants.ts b/tests/e2e/ui/constants.ts index 9d918736262..d2e7906df2c 100644 --- a/tests/e2e/ui/constants.ts +++ b/tests/e2e/ui/constants.ts @@ -24,20 +24,40 @@ export const INTERNAL_USER_STORAGE_PATH = storagePath("internalUser.storageState export const INTERNAL_VIEWER_STORAGE_PATH = storagePath("internalViewer.storageState.json"); export const TEAM_ADMIN_STORAGE_PATH = storagePath("teamAdmin.storageState.json"); -// Seeded user identities (match seed.sql) +// Seeded user identities (match fixtures/seed.ts) export const E2E_PROXY_ADMIN_USER_ID = "e2e-proxy-admin"; export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local"; +export const E2E_ADMIN_VIEWER_USER_ID = "e2e-admin-viewer"; +export const E2E_ADMIN_VIEWER_EMAIL = "adminviewer@test.local"; export const E2E_INTERNAL_USER_ID = "e2e-internal-user"; export const E2E_INTERNAL_USER_EMAIL = "internal@test.local"; +export const E2E_INTERNAL_VIEWER_USER_ID = "e2e-internal-viewer"; +export const E2E_INTERNAL_VIEWER_EMAIL = "viewer@test.local"; +export const E2E_TEAM_ADMIN_USER_ID = "e2e-team-admin"; +export const E2E_TEAM_ADMIN_EMAIL = "teamadmin@test.local"; +export const E2E_INVITABLE_USER_ID = "e2e-invitable-user"; +export const E2E_INVITABLE_USER_EMAIL = "invitable@test.local"; +export const E2E_INTERNAL_NOTEAM_USER_ID = "e2e-internal-noteam"; +export const E2E_INTERNAL_NOTEAM_EMAIL = "noteam@test.local"; +export const E2E_INVITABLE_BY_TEAM_ADMIN_USER_ID = "e2e-invitable-by-team-admin"; +export const E2E_INVITABLE_BY_TEAM_ADMIN_EMAIL = "invitable-team@test.local"; +export const E2E_REMOVABLE_MEMBER_USER_ID = "e2e-removable-member"; +export const E2E_REMOVABLE_MEMBER_EMAIL = "removable@test.local"; +export const E2E_USER_PASSWORD = "test"; -// Key aliases for seeded test keys (match seed.sql) +// Organization and its budget (match fixtures/seed.ts) +export const E2E_ORG_ID = "e2e-org-main"; +export const E2E_ORG_ALIAS = "E2E Organization"; +export const E2E_ORG_BUDGET_ID = "e2e-budget-org"; + +// Key aliases for seeded test keys (match fixtures/seed.ts) export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey"; export const E2E_DELETE_KEY_ALIAS = "e2eDeleteKey"; export const E2E_REGENERATE_KEY_ALIAS = "e2eRegenerateKey"; export const E2E_INTERNAL_USER_KEY_ALIAS = "e2eInternalUserKey"; export const E2E_VIEWER_KEY_ALIAS = "e2eViewerKey"; -// Team identifiers (match seed.sql) +// Team identifiers (match fixtures/seed.ts) export const E2E_TEAM_CRUD_ID = "e2e-team-crud"; export const E2E_TEAM_CRUD_ALIAS = "E2E Team CRUD"; export const E2E_TEAM_DELETE_ID = "e2e-team-delete"; diff --git a/tests/e2e/ui/fixtures/seed.sql b/tests/e2e/ui/fixtures/seed.sql deleted file mode 100644 index a1218633cdb..00000000000 --- a/tests/e2e/ui/fixtures/seed.sql +++ /dev/null @@ -1,86 +0,0 @@ --- E2E Test Seed Data --- Idempotent: deletes all e2e-* rows then re-inserts deterministic data. - --- 1. Clean up in dependency order -DELETE FROM "LiteLLM_TeamMembership" WHERE "user_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_VerificationToken" WHERE token LIKE 'e2e-%'; -DELETE FROM "LiteLLM_TeamTable" WHERE "team_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_OrganizationTable" WHERE "organization_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_UserTable" WHERE "user_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_BudgetTable" WHERE "budget_id" LIKE 'e2e-%'; - --- 2. Budget (created_by and updated_by are NOT NULL) -INSERT INTO "LiteLLM_BudgetTable" ("budget_id", "max_budget", "created_by", "updated_by") -VALUES ('e2e-budget-org', 1000, 'e2e-proxy-admin', 'e2e-proxy-admin'); - --- 3. Organization (created_by and updated_by are NOT NULL) -INSERT INTO "LiteLLM_OrganizationTable" ( - "organization_id", "organization_alias", "budget_id", - "metadata", "models", "spend", "model_spend", - "created_by", "updated_by" -) VALUES ( - 'e2e-org-main', 'E2E Organization', 'e2e-budget-org', - '{}'::jsonb, ARRAY[]::text[], 0.0, '{}'::jsonb, - 'e2e-proxy-admin', 'e2e-proxy-admin' -); - --- 4. Users (password hash is scrypt of "test") -INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password") -VALUES - ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'); - --- 5. Teams (members_with_roles is required JSON) -INSERT INTO "LiteLLM_TeamTable" ( - "team_id", "team_alias", "organization_id", "admins", "members", - "members_with_roles", "metadata", "models", "spend", "model_spend", "model_max_budget", "blocked" -) VALUES - ('e2e-team-crud', 'E2E Team CRUD', NULL, - '{"e2e-team-admin"}', - '{"e2e-team-admin","e2e-internal-user","e2e-internal-viewer","e2e-removable-member"}', - '[{"role":"admin","user_id":"e2e-team-admin"},{"role":"user","user_id":"e2e-internal-user"},{"role":"user","user_id":"e2e-internal-viewer"},{"role":"user","user_id":"e2e-removable-member"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4","fake-anthropic-claude"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-delete', 'E2E Team Delete', NULL, - '{"e2e-team-admin"}', '{"e2e-team-admin"}', - '[{"role":"admin","user_id":"e2e-team-admin"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-org', 'E2E Team In Org', 'e2e-org-main', - '{}', '{"e2e-internal-user"}', - '[{"role":"user","user_id":"e2e-internal-user"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-no-admin', 'E2E Team No Admin', NULL, - '{}', '{"e2e-invitable-user"}', - '[{"role":"user","user_id":"e2e-invitable-user"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false); - --- 6. Team Memberships (only user_id, team_id, spend — no created_at/updated_at) -INSERT INTO "LiteLLM_TeamMembership" ("user_id", "team_id", "spend") -VALUES - ('e2e-team-admin', 'e2e-team-crud', 0.0), - ('e2e-internal-user', 'e2e-team-crud', 0.0), - ('e2e-internal-viewer', 'e2e-team-crud', 0.0), - ('e2e-removable-member', 'e2e-team-crud', 0.0), - ('e2e-team-admin', 'e2e-team-delete', 0.0), - ('e2e-internal-user', 'e2e-team-org', 0.0), - ('e2e-invitable-user', 'e2e-team-no-admin', 0.0); - --- 7. Verification Tokens (API Keys) -INSERT INTO "LiteLLM_VerificationToken" ( - "token", "key_name", "key_alias", "user_id", "team_id", - "models", "spend", "max_budget", "expires", "metadata" -) VALUES - ('e2e-key-update-limits', 'sk-e2e-update', 'e2eUpdateLimitsKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-delete', 'sk-e2e-delete', 'e2eDeleteKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-regenerate', 'sk-e2e-regen', 'e2eRegenerateKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-internal-user', 'sk-e2e-internal', 'e2eInternalUserKey', 'e2e-internal-user', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-viewer', 'sk-e2e-viewer', 'e2eViewerKey', 'e2e-internal-viewer', NULL, '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb); diff --git a/tests/e2e/ui/fixtures/seed.ts b/tests/e2e/ui/fixtures/seed.ts new file mode 100644 index 00000000000..707c7b7bafd --- /dev/null +++ b/tests/e2e/ui/fixtures/seed.ts @@ -0,0 +1,272 @@ +import type { APIRequestContext } from "@playwright/test"; +import { + E2E_ADMIN_VIEWER_EMAIL, + E2E_ADMIN_VIEWER_USER_ID, + E2E_DELETE_KEY_ALIAS, + E2E_INTERNAL_NOTEAM_EMAIL, + E2E_INTERNAL_NOTEAM_USER_ID, + E2E_INTERNAL_USER_EMAIL, + E2E_INTERNAL_USER_ID, + E2E_INTERNAL_USER_KEY_ALIAS, + E2E_INTERNAL_VIEWER_EMAIL, + E2E_INTERNAL_VIEWER_USER_ID, + E2E_INVITABLE_BY_TEAM_ADMIN_EMAIL, + E2E_INVITABLE_BY_TEAM_ADMIN_USER_ID, + E2E_INVITABLE_USER_EMAIL, + E2E_INVITABLE_USER_ID, + E2E_ORG_ALIAS, + E2E_ORG_BUDGET_ID, + E2E_ORG_ID, + E2E_PROXY_ADMIN_EMAIL, + E2E_PROXY_ADMIN_USER_ID, + E2E_REGENERATE_KEY_ALIAS, + E2E_REMOVABLE_MEMBER_EMAIL, + E2E_REMOVABLE_MEMBER_USER_ID, + E2E_TEAM_ADMIN_EMAIL, + E2E_TEAM_ADMIN_USER_ID, + E2E_TEAM_CRUD_ALIAS, + E2E_TEAM_CRUD_ID, + E2E_TEAM_DELETE_ALIAS, + E2E_TEAM_DELETE_ID, + E2E_TEAM_NO_ADMIN_ALIAS, + E2E_TEAM_NO_ADMIN_ID, + E2E_TEAM_ORG_ALIAS, + E2E_TEAM_ORG_ID, + E2E_UPDATE_LIMITS_KEY_ALIAS, + E2E_USER_PASSWORD, + E2E_VIEWER_KEY_ALIAS, +} from "../constants"; + +type UserRole = "proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer"; +type TeamMemberRole = "admin" | "user"; + +type SeedUser = { readonly userId: string; readonly email: string; readonly role: UserRole }; +type SeedTeamMember = { readonly user_id: string; readonly role: TeamMemberRole }; +type SeedTeam = { + readonly teamId: string; + readonly alias: string; + readonly organizationId: string | null; + readonly models: readonly string[]; + readonly members: readonly SeedTeamMember[]; +}; +type SeedKey = { + readonly alias: string; + readonly userId: string; + readonly teamId: string | null; + readonly models: readonly string[]; +}; + +const OPENAI_MODEL = "fake-openai-gpt-4"; +const ANTHROPIC_MODEL = "fake-anthropic-claude"; +const ORG_MAX_BUDGET = 1000; + +// The master key's own user is auto-added as an admin of every team it creates, +// which would make the proxy admin a member of teams the suite needs it to be a +// stranger to. Removed after each team is created. +const MASTER_KEY_USER_ID = "default_user_id"; + +const USERS: readonly SeedUser[] = [ + { userId: E2E_PROXY_ADMIN_USER_ID, email: E2E_PROXY_ADMIN_EMAIL, role: "proxy_admin" }, + { userId: E2E_ADMIN_VIEWER_USER_ID, email: E2E_ADMIN_VIEWER_EMAIL, role: "proxy_admin_viewer" }, + { userId: E2E_INTERNAL_USER_ID, email: E2E_INTERNAL_USER_EMAIL, role: "internal_user" }, + { userId: E2E_INTERNAL_VIEWER_USER_ID, email: E2E_INTERNAL_VIEWER_EMAIL, role: "internal_user_viewer" }, + { userId: E2E_TEAM_ADMIN_USER_ID, email: E2E_TEAM_ADMIN_EMAIL, role: "internal_user" }, + { userId: E2E_INVITABLE_USER_ID, email: E2E_INVITABLE_USER_EMAIL, role: "internal_user" }, + { userId: E2E_INTERNAL_NOTEAM_USER_ID, email: E2E_INTERNAL_NOTEAM_EMAIL, role: "internal_user" }, + { + userId: E2E_INVITABLE_BY_TEAM_ADMIN_USER_ID, + email: E2E_INVITABLE_BY_TEAM_ADMIN_EMAIL, + role: "internal_user", + }, + { userId: E2E_REMOVABLE_MEMBER_USER_ID, email: E2E_REMOVABLE_MEMBER_EMAIL, role: "internal_user" }, +]; + +const TEAMS: readonly SeedTeam[] = [ + { + teamId: E2E_TEAM_CRUD_ID, + alias: E2E_TEAM_CRUD_ALIAS, + organizationId: null, + models: [OPENAI_MODEL, ANTHROPIC_MODEL], + members: [ + { user_id: E2E_TEAM_ADMIN_USER_ID, role: "admin" }, + { user_id: E2E_INTERNAL_USER_ID, role: "user" }, + { user_id: E2E_INTERNAL_VIEWER_USER_ID, role: "user" }, + { user_id: E2E_REMOVABLE_MEMBER_USER_ID, role: "user" }, + ], + }, + { + teamId: E2E_TEAM_DELETE_ID, + alias: E2E_TEAM_DELETE_ALIAS, + organizationId: null, + models: [OPENAI_MODEL], + members: [{ user_id: E2E_TEAM_ADMIN_USER_ID, role: "admin" }], + }, + { + teamId: E2E_TEAM_ORG_ID, + alias: E2E_TEAM_ORG_ALIAS, + organizationId: E2E_ORG_ID, + models: [OPENAI_MODEL], + members: [{ user_id: E2E_INTERNAL_USER_ID, role: "user" }], + }, + { + teamId: E2E_TEAM_NO_ADMIN_ID, + alias: E2E_TEAM_NO_ADMIN_ALIAS, + organizationId: null, + models: [OPENAI_MODEL], + members: [{ user_id: E2E_INVITABLE_USER_ID, role: "user" }], + }, +]; + +const KEYS: readonly SeedKey[] = [ + { + alias: E2E_UPDATE_LIMITS_KEY_ALIAS, + userId: E2E_PROXY_ADMIN_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_DELETE_KEY_ALIAS, + userId: E2E_PROXY_ADMIN_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_REGENERATE_KEY_ALIAS, + userId: E2E_PROXY_ADMIN_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_INTERNAL_USER_KEY_ALIAS, + userId: E2E_INTERNAL_USER_ID, + teamId: E2E_TEAM_CRUD_ID, + models: [OPENAI_MODEL], + }, + { + alias: E2E_VIEWER_KEY_ALIAS, + userId: E2E_INTERNAL_VIEWER_USER_ID, + teamId: null, + models: [OPENAI_MODEL], + }, +]; + +type JsonBody = Record; + +const jsonHeaders = (masterKey: string): Record => ({ + Authorization: `Bearer ${masterKey}`, + "Content-Type": "application/json", +}); + +async function post( + api: APIRequestContext, + url: string, + data: JsonBody, + headers: Record, +): Promise { + const res = await api.post(url, { headers, data }); + if (!res.ok()) { + throw new Error(`Seeding call POST ${url} failed (${res.status()}): ${await res.text()}`); + } +} + +async function postAllowingMissing( + api: APIRequestContext, + url: string, + data: JsonBody, + headers: Record, +): Promise { + const res = await api.post(url, { headers, data }); + if (!res.ok() && res.status() !== 404) { + throw new Error(`Seeding call POST ${url} failed (${res.status()}): ${await res.text()}`); + } +} + +async function deleteAllowingMissing( + api: APIRequestContext, + url: string, + data: JsonBody, + headers: Record, +): Promise { + const res = await api.delete(url, { headers, data }); + if (!res.ok() && res.status() !== 404) { + throw new Error(`Seeding call DELETE ${url} failed (${res.status()}): ${await res.text()}`); + } +} + +// /team/delete and /user/delete are all-or-nothing: one unknown id in the batch +// aborts the whole call and deletes nothing, so each id goes in its own request. +async function removeFixtures( + api: APIRequestContext, + apiBase: string, + headers: Record, +): Promise { + await postAllowingMissing(api, `${apiBase}/key/delete`, { key_aliases: KEYS.map((key) => key.alias) }, headers); + for (const team of TEAMS) { + await postAllowingMissing(api, `${apiBase}/team/delete`, { team_ids: [team.teamId] }, headers); + } + for (const user of USERS) { + await postAllowingMissing(api, `${apiBase}/user/delete`, { user_ids: [user.userId] }, headers); + } + await deleteAllowingMissing(api, `${apiBase}/organization/delete`, { organization_ids: [E2E_ORG_ID] }, headers); + await postAllowingMissing(api, `${apiBase}/budget/delete`, { id: E2E_ORG_BUDGET_ID }, headers); +} + +async function createFixtures( + api: APIRequestContext, + apiBase: string, + headers: Record, +): Promise { + await post(api, `${apiBase}/budget/new`, { budget_id: E2E_ORG_BUDGET_ID, max_budget: ORG_MAX_BUDGET }, headers); + await post( + api, + `${apiBase}/organization/new`, + { organization_id: E2E_ORG_ID, organization_alias: E2E_ORG_ALIAS, budget_id: E2E_ORG_BUDGET_ID }, + headers, + ); + + for (const user of USERS) { + await post( + api, + `${apiBase}/user/new`, + { user_id: user.userId, user_email: user.email, user_role: user.role, auto_create_key: false }, + headers, + ); + await post(api, `${apiBase}/user/update`, { user_id: user.userId, password: E2E_USER_PASSWORD }, headers); + } + + for (const team of TEAMS) { + await post( + api, + `${apiBase}/team/new`, + { + team_id: team.teamId, + team_alias: team.alias, + organization_id: team.organizationId, + models: team.models, + members_with_roles: team.members, + }, + headers, + ); + await postAllowingMissing( + api, + `${apiBase}/team/member_delete`, + { team_id: team.teamId, user_id: MASTER_KEY_USER_ID }, + headers, + ); + } + + for (const key of KEYS) { + await post( + api, + `${apiBase}/key/generate`, + { key_alias: key.alias, user_id: key.userId, team_id: key.teamId, models: key.models }, + headers, + ); + } +} + +export async function seedFixtures(api: APIRequestContext, apiBase: string, masterKey: string): Promise { + const headers = jsonHeaders(masterKey); + await removeFixtures(api, apiBase, headers); + await createFixtures(api, apiBase, headers); +} diff --git a/tests/e2e/ui/fixtures/users.ts b/tests/e2e/ui/fixtures/users.ts index 731234b5ea9..b92ded48c41 100644 --- a/tests/e2e/ui/fixtures/users.ts +++ b/tests/e2e/ui/fixtures/users.ts @@ -1,6 +1,11 @@ import { ADMIN_STORAGE_PATH, ADMIN_VIEWER_STORAGE_PATH, + E2E_ADMIN_VIEWER_EMAIL, + E2E_INTERNAL_USER_EMAIL, + E2E_INTERNAL_VIEWER_EMAIL, + E2E_TEAM_ADMIN_EMAIL, + E2E_USER_PASSWORD, INTERNAL_USER_STORAGE_PATH, INTERNAL_VIEWER_STORAGE_PATH, TEAM_ADMIN_STORAGE_PATH, @@ -20,20 +25,20 @@ export const users: Record = { password: process.env.LITELLM_MASTER_KEY || "sk-1234", }, [Role.ProxyAdminViewer]: { - email: "adminviewer@test.local", - password: "test", + email: E2E_ADMIN_VIEWER_EMAIL, + password: E2E_USER_PASSWORD, }, [Role.InternalUser]: { - email: "internal@test.local", - password: "test", + email: E2E_INTERNAL_USER_EMAIL, + password: E2E_USER_PASSWORD, }, [Role.InternalUserViewer]: { - email: "viewer@test.local", - password: "test", + email: E2E_INTERNAL_VIEWER_EMAIL, + password: E2E_USER_PASSWORD, }, [Role.TeamAdmin]: { - email: "teamadmin@test.local", - password: "test", + email: E2E_TEAM_ADMIN_EMAIL, + password: E2E_USER_PASSWORD, }, }; diff --git a/tests/e2e/ui/globalSetup.ts b/tests/e2e/ui/globalSetup.ts index 6068a51c88e..021916fd03a 100644 --- a/tests/e2e/ui/globalSetup.ts +++ b/tests/e2e/ui/globalSetup.ts @@ -1,5 +1,6 @@ import { chromium, expect, request } from "@playwright/test"; import { users, Role, STORAGE_PATHS } from "./fixtures/users"; +import { seedFixtures } from "./fixtures/seed"; import { ARTIFACT_DIR, UI_BASE_URL } from "./constants"; import * as fs from "fs"; import * as path from "path"; @@ -29,6 +30,12 @@ async function globalSetup() { if (!settingsRes.ok()) { throw new Error(`Enabling enable_projects_ui failed (${settingsRes.status()}): ${await settingsRes.text()}`); } + + // The users, teams and keys every spec reads are created here over the + // management API rather than by loading SQL into the database, so the suite + // can run against any deployment it can reach (a local proxy, CI, or a remote + // one whose database it has no direct access to). + await seedFixtures(api, `${UI_BASE_URL}${rootPath}`, masterKey); await api.dispose(); for (const role of Object.values(Role)) { diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index 858eb401c8e..1bc1cdd2034 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -3,7 +3,8 @@ set -euo pipefail # ================================================================ # UI E2E Test Runner (Consolidated) -# Starts postgres, seeds DB, starts mock + proxy, runs Playwright. +# Starts postgres, starts mock + proxy, runs Playwright (which seeds +# its fixtures over the management API in globalSetup). # All tests target the proxy on port 4000 (which serves both API # and UI from the built Next.js static export). # @@ -56,7 +57,7 @@ done # --- Database setup --- if [ "$IS_CI" = "false" ]; then - for cmd in docker psql; do + for cmd in docker pg_isready; do command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } done for port in 4000 5432 8090; do @@ -174,17 +175,6 @@ if [ "$PROXY_READY" -ne 1 ]; then fi echo "Proxy is ready." -# --- Seed database --- -echo "=== Seeding database ===" -DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p') -DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') -DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p') -DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p') -DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p') - -PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ - -f "$SCRIPT_DIR/fixtures/seed.sql" - # --- Playwright --- echo "=== Installing Playwright dependencies ===" cd "$SCRIPT_DIR" diff --git a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts index 18b43ec89b2..a05e2572760 100644 --- a/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts +++ b/tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from "@playwright/test"; import { E2E_INTERNAL_USER_KEY_ALIAS, + E2E_REMOVABLE_MEMBER_USER_ID, E2E_TEAM_CRUD_ALIAS, E2E_TEAM_CRUD_ID, TEAM_ADMIN_STORAGE_PATH, @@ -71,7 +72,7 @@ test.describe("Team Admin", () => { // Seeded members appear in the roster by user_id (members_with_roles has no // email), so match the row on the user_id rather than the email. - const row = page.locator("tr", { hasText: "e2e-removable-member" }).first(); + const row = page.locator("tr", { hasText: E2E_REMOVABLE_MEMBER_USER_ID }).first(); await expect(row).toBeVisible({ timeout: 10_000 }); await row.getByTestId("delete-member").click();