diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f01b6de4f3..7a071fc2a16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2776,20 +2776,10 @@ jobs: - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - run: name: Start LiteLLM proxy environment: LITELLM_MASTER_KEY: "sk-1234" - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" SERVER_ROOT_PATH: "" # PROXY_LOGOUT_URL is inherited from the job-level environment so the @@ -2904,20 +2894,10 @@ jobs: - run: name: Push Prisma schema command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - run: name: Start LiteLLM proxy under a server root path environment: LITELLM_MASTER_KEY: "sk-1234" - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" DISABLE_SCHEMA_UPDATE: "true" # Output flows to this step's own log, so a boot crash is visible here # rather than swallowed by a downstream readiness probe. diff --git a/pyrightconfig.json b/pyrightconfig.json index 2686ccd73d9..eabfbf515c4 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,7 +1,7 @@ { "include": ["litellm"], "ignore": [], - "exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "tests/e2e/ui", "litellm/types/utils.py", "litellm/proxy/_types.py"], + "exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "litellm/types/utils.py", "litellm/proxy/_types.py"], "pythonVersion": "3.12", "typeCheckingMode": "strict", "enableTypeIgnoreComments": false, diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 53a2437460b..81e0c4d857e 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke -- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` +- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard at `LITELLM_PROXY_URL` (default `http://localhost:4000`), the same live gateway the python suites target. The gateway only needs a master key and a database (`store_model_in_db: true`); `globalSetup` provisions everything else (e2e-* users, teams, keys, and real-provider model deployments) through the management API and `run_e2e.sh` can stand up a disposable gateway when none is running. The suite runs single-worker and needs the gateway to itself while it runs: several specs mutate global gateway state (`router_settings` via `/config/update`, the public model groups list, `enable_projects_ui`), so do not run the python suites against the same gateway at the same time. It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it ## MCP suite: real Datadog only diff --git a/tests/e2e/ui/constants.ts b/tests/e2e/ui/constants.ts index 236909384b0..623af06e996 100644 --- a/tests/e2e/ui/constants.ts +++ b/tests/e2e/ui/constants.ts @@ -1,3 +1,8 @@ +export const PROXY_BASE_URL = (process.env.LITELLM_PROXY_URL ?? "http://localhost:4000").replace(/\/+$/, ""); + +export const E2E_UI_OPENAI_MODEL = "e2e-ui-openai"; +export const E2E_UI_ANTHROPIC_MODEL = "e2e-ui-anthropic"; + // Storage state paths for each role export const ADMIN_STORAGE_PATH = "admin.storageState.json"; export const ADMIN_VIEWER_STORAGE_PATH = "adminViewer.storageState.json"; @@ -5,20 +10,20 @@ export const INTERNAL_USER_STORAGE_PATH = "internalUser.storageState.json"; export const INTERNAL_VIEWER_STORAGE_PATH = "internalViewer.storageState.json"; export const TEAM_ADMIN_STORAGE_PATH = "teamAdmin.storageState.json"; -// Seeded user identities (match seed.sql) +// Seeded user identities (match fixtures/apiSeed.ts) export const E2E_PROXY_ADMIN_USER_ID = "e2e-proxy-admin"; export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local"; export const E2E_INTERNAL_USER_ID = "e2e-internal-user"; export const E2E_INTERNAL_USER_EMAIL = "internal@test.local"; -// Key aliases for seeded test keys (match seed.sql) +// Key aliases for seeded test keys (match fixtures/apiSeed.ts) export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey"; export const E2E_DELETE_KEY_ALIAS = "e2eDeleteKey"; export const E2E_REGENERATE_KEY_ALIAS = "e2eRegenerateKey"; export const E2E_INTERNAL_USER_KEY_ALIAS = "e2eInternalUserKey"; export const E2E_VIEWER_KEY_ALIAS = "e2eViewerKey"; -// Team identifiers (match seed.sql) +// Team identifiers (match fixtures/apiSeed.ts) export const E2E_TEAM_CRUD_ID = "e2e-team-crud"; export const E2E_TEAM_CRUD_ALIAS = "E2E Team CRUD"; export const E2E_TEAM_DELETE_ID = "e2e-team-delete"; diff --git a/tests/e2e/ui/fixtures/apiSeed.ts b/tests/e2e/ui/fixtures/apiSeed.ts new file mode 100644 index 00000000000..eb8883e9c72 --- /dev/null +++ b/tests/e2e/ui/fixtures/apiSeed.ts @@ -0,0 +1,130 @@ +import { APIRequestContext, request } from "@playwright/test"; +import { E2E_UI_OPENAI_MODEL, E2E_UI_ANTHROPIC_MODEL } from "../constants"; + +const OPENAI_UPSTREAM = `openai/${process.env.E2E_CHEAP_OPENAI_MODEL ?? "gpt-5.5"}`; +const ANTHROPIC_UPSTREAM = `anthropic/${process.env.E2E_CHEAP_ANTHROPIC_MODEL ?? "claude-haiku-4-5"}`; + +const SEED_PASSWORD = "test"; + +const MODELS = [ + { + model_name: E2E_UI_OPENAI_MODEL, + litellm_params: { model: OPENAI_UPSTREAM, api_key: "os.environ/OPENAI_API_KEY" }, + model_info: { id: E2E_UI_OPENAI_MODEL }, + }, + { + model_name: E2E_UI_ANTHROPIC_MODEL, + litellm_params: { model: ANTHROPIC_UPSTREAM, api_key: "os.environ/ANTHROPIC_API_KEY" }, + model_info: { id: E2E_UI_ANTHROPIC_MODEL }, + }, +] as const; + +const ORG = { organization_id: "e2e-org-main", organization_alias: "E2E Organization", max_budget: 1000 } as const; + +const USERS = [ + { user_id: "e2e-proxy-admin", user_email: "admin@test.local", user_role: "proxy_admin" }, + { user_id: "e2e-admin-viewer", user_email: "adminviewer@test.local", user_role: "proxy_admin_viewer" }, + { user_id: "e2e-internal-user", user_email: "internal@test.local", user_role: "internal_user" }, + { user_id: "e2e-internal-viewer", user_email: "viewer@test.local", user_role: "internal_user_viewer" }, + { user_id: "e2e-team-admin", user_email: "teamadmin@test.local", user_role: "internal_user" }, + { user_id: "e2e-invitable-user", user_email: "invitable@test.local", user_role: "internal_user" }, + { user_id: "e2e-internal-noteam", user_email: "noteam@test.local", user_role: "internal_user" }, + { user_id: "e2e-invitable-by-team-admin", user_email: "invitable-team@test.local", user_role: "internal_user" }, + { user_id: "e2e-removable-member", user_email: "removable@test.local", user_role: "internal_user" }, +] as const; + +const TEAMS = [ + { + team_id: "e2e-team-crud", + team_alias: "E2E Team CRUD", + organization_id: null, + models: [E2E_UI_OPENAI_MODEL, E2E_UI_ANTHROPIC_MODEL], + members_with_roles: [ + { role: "admin", user_id: "e2e-team-admin" }, + { role: "user", user_id: "e2e-internal-user" }, + { role: "user", user_id: "e2e-internal-viewer" }, + { role: "user", user_id: "e2e-removable-member" }, + ], + }, + { + team_id: "e2e-team-delete", + team_alias: "E2E Team Delete", + organization_id: null, + models: [E2E_UI_OPENAI_MODEL], + members_with_roles: [{ role: "admin", user_id: "e2e-team-admin" }], + }, + { + team_id: "e2e-team-org", + team_alias: "E2E Team In Org", + organization_id: ORG.organization_id, + models: [E2E_UI_OPENAI_MODEL], + members_with_roles: [{ role: "user", user_id: "e2e-internal-user" }], + }, + { + team_id: "e2e-team-no-admin", + team_alias: "E2E Team No Admin", + organization_id: null, + models: [E2E_UI_OPENAI_MODEL], + members_with_roles: [{ role: "user", user_id: "e2e-invitable-user" }], + }, +] as const; + +const KEYS = [ + { key_alias: "e2eUpdateLimitsKey", user_id: "e2e-proxy-admin", team_id: "e2e-team-crud" }, + { key_alias: "e2eDeleteKey", user_id: "e2e-proxy-admin", team_id: "e2e-team-crud" }, + { key_alias: "e2eRegenerateKey", user_id: "e2e-proxy-admin", team_id: "e2e-team-crud" }, + { key_alias: "e2eInternalUserKey", user_id: "e2e-internal-user", team_id: "e2e-team-crud" }, + { key_alias: "e2eViewerKey", user_id: "e2e-internal-viewer", team_id: null }, +] as const; + +async function post(api: APIRequestContext, path: string, data: unknown, allowFailure = false): Promise { + const res = await api.post(path, { data }); + if (res.ok() || allowFailure) return; + throw new Error(`Seeding ${path} failed (${res.status()}): ${await res.text()}`); +} + +async function teardown(api: APIRequestContext): Promise { + for (const key of KEYS) { + await post(api, "/key/delete", { key_aliases: [key.key_alias] }, true); + } + for (const team of TEAMS) { + await post(api, "/team/delete", { team_ids: [team.team_id] }, true); + } + for (const user of USERS) { + await post(api, "/user/delete", { user_ids: [user.user_id] }, true); + } + await api.delete("/organization/delete", { data: { organization_ids: [ORG.organization_id] } }); + for (const model of MODELS) { + await post(api, "/model/delete", { id: model.model_info.id }, true); + } +} + +async function create(api: APIRequestContext): Promise { + for (const model of MODELS) { + await post(api, "/model/new", model); + } + await post(api, "/organization/new", ORG); + for (const user of USERS) { + await post(api, "/user/new", { ...user, auto_create_key: false }); + await post(api, "/user/update", { user_id: user.user_id, password: SEED_PASSWORD }); + } + for (const team of TEAMS) { + await post(api, "/team/new", team); + } + for (const key of KEYS) { + await post(api, "/key/generate", { ...key, models: [E2E_UI_OPENAI_MODEL] }); + } +} + +export async function seedGateway(baseUrl: string, masterKey: string): Promise { + const api = await request.newContext({ + baseURL: baseUrl, + extraHTTPHeaders: { Authorization: `Bearer ${masterKey}` }, + }); + try { + await teardown(api); + await create(api); + } finally { + await api.dispose(); + } +} diff --git a/tests/e2e/ui/fixtures/config.yml b/tests/e2e/ui/fixtures/config.yml index 3d250984bca..7312d15bb05 100644 --- a/tests/e2e/ui/fixtures/config.yml +++ b/tests/e2e/ui/fixtures/config.yml @@ -1,15 +1,3 @@ -model_list: - - model_name: fake-openai-gpt-4 - litellm_params: - model: openai/fake-gpt-4 - api_base: os.environ/MOCK_LLM_URL - api_key: fake-key - - model_name: fake-anthropic-claude - litellm_params: - model: openai/fake-claude - api_base: os.environ/MOCK_LLM_URL - api_key: fake-key - general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL diff --git a/tests/e2e/ui/fixtures/mock_llm_server/server.py b/tests/e2e/ui/fixtures/mock_llm_server/server.py deleted file mode 100644 index 8e92065c696..00000000000 --- a/tests/e2e/ui/fixtures/mock_llm_server/server.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Mock LLM server for UI e2e tests. -Responds to OpenAI-format endpoints with canned responses. -""" - -import time -import json -import uuid - -import uvicorn -from fastapi import FastAPI, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse - - -app = FastAPI(title="Mock LLM Server") -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_methods=["*"], - allow_headers=["*"], -) - - -@app.get("/health") -async def health(): - return {"status": "ok"} - - -@app.get("/v1/models") -@app.get("/models") -async def list_models(): - return { - "object": "list", - "data": [ - {"id": "fake-gpt-4", "object": "model", "owned_by": "mock"}, - {"id": "fake-claude", "object": "model", "owned_by": "mock"}, - ], - } - - -@app.post("/v1/chat/completions") -@app.post("/chat/completions") -async def chat_completions(request: Request): - body = await request.json() - model = body.get("model", "mock-model") - stream = body.get("stream", False) - - response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" - created = int(time.time()) - - if stream: - - async def stream_generator(): - chunk = { - "id": response_id, - "object": "chat.completion.chunk", - "created": created, - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "content": "This is a mock response.", - }, - "finish_reason": None, - } - ], - } - yield f"data: {json.dumps(chunk)}\n\n" - - done_chunk = { - "id": response_id, - "object": "chat.completion.chunk", - "created": created, - "model": model, - "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], - } - yield f"data: {json.dumps(done_chunk)}\n\n" - yield "data: [DONE]\n\n" - - return StreamingResponse(stream_generator(), media_type="text/event-stream") - - return { - "id": response_id, - "object": "chat.completion", - "created": created, - "model": model, - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": "This is a mock response."}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18}, - } - - -@app.post("/v1/embeddings") -@app.post("/embeddings") -async def embeddings(request: Request): - body = await request.json() - inputs = body.get("input", [""]) - if isinstance(inputs, str): - inputs = [inputs] - return { - "object": "list", - "data": [ - {"object": "embedding", "index": i, "embedding": [0.0] * 1536} - for i in range(len(inputs)) - ], - "model": body.get("model", "mock-embedding"), - "usage": {"prompt_tokens": 5, "total_tokens": 5}, - } - - -if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=8090) diff --git a/tests/e2e/ui/fixtures/seed.sql b/tests/e2e/ui/fixtures/seed.sql deleted file mode 100644 index a1218633cdb..00000000000 --- a/tests/e2e/ui/fixtures/seed.sql +++ /dev/null @@ -1,86 +0,0 @@ --- E2E Test Seed Data --- Idempotent: deletes all e2e-* rows then re-inserts deterministic data. - --- 1. Clean up in dependency order -DELETE FROM "LiteLLM_TeamMembership" WHERE "user_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_VerificationToken" WHERE token LIKE 'e2e-%'; -DELETE FROM "LiteLLM_TeamTable" WHERE "team_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_OrganizationTable" WHERE "organization_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_UserTable" WHERE "user_id" LIKE 'e2e-%'; -DELETE FROM "LiteLLM_BudgetTable" WHERE "budget_id" LIKE 'e2e-%'; - --- 2. Budget (created_by and updated_by are NOT NULL) -INSERT INTO "LiteLLM_BudgetTable" ("budget_id", "max_budget", "created_by", "updated_by") -VALUES ('e2e-budget-org', 1000, 'e2e-proxy-admin', 'e2e-proxy-admin'); - --- 3. Organization (created_by and updated_by are NOT NULL) -INSERT INTO "LiteLLM_OrganizationTable" ( - "organization_id", "organization_alias", "budget_id", - "metadata", "models", "spend", "model_spend", - "created_by", "updated_by" -) VALUES ( - 'e2e-org-main', 'E2E Organization', 'e2e-budget-org', - '{}'::jsonb, ARRAY[]::text[], 0.0, '{}'::jsonb, - 'e2e-proxy-admin', 'e2e-proxy-admin' -); - --- 4. Users (password hash is scrypt of "test") -INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password") -VALUES - ('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'), - ('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'); - --- 5. Teams (members_with_roles is required JSON) -INSERT INTO "LiteLLM_TeamTable" ( - "team_id", "team_alias", "organization_id", "admins", "members", - "members_with_roles", "metadata", "models", "spend", "model_spend", "model_max_budget", "blocked" -) VALUES - ('e2e-team-crud', 'E2E Team CRUD', NULL, - '{"e2e-team-admin"}', - '{"e2e-team-admin","e2e-internal-user","e2e-internal-viewer","e2e-removable-member"}', - '[{"role":"admin","user_id":"e2e-team-admin"},{"role":"user","user_id":"e2e-internal-user"},{"role":"user","user_id":"e2e-internal-viewer"},{"role":"user","user_id":"e2e-removable-member"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4","fake-anthropic-claude"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-delete', 'E2E Team Delete', NULL, - '{"e2e-team-admin"}', '{"e2e-team-admin"}', - '[{"role":"admin","user_id":"e2e-team-admin"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-org', 'E2E Team In Org', 'e2e-org-main', - '{}', '{"e2e-internal-user"}', - '[{"role":"user","user_id":"e2e-internal-user"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false), - - ('e2e-team-no-admin', 'E2E Team No Admin', NULL, - '{}', '{"e2e-invitable-user"}', - '[{"role":"user","user_id":"e2e-invitable-user"}]'::jsonb, - '{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false); - --- 6. Team Memberships (only user_id, team_id, spend — no created_at/updated_at) -INSERT INTO "LiteLLM_TeamMembership" ("user_id", "team_id", "spend") -VALUES - ('e2e-team-admin', 'e2e-team-crud', 0.0), - ('e2e-internal-user', 'e2e-team-crud', 0.0), - ('e2e-internal-viewer', 'e2e-team-crud', 0.0), - ('e2e-removable-member', 'e2e-team-crud', 0.0), - ('e2e-team-admin', 'e2e-team-delete', 0.0), - ('e2e-internal-user', 'e2e-team-org', 0.0), - ('e2e-invitable-user', 'e2e-team-no-admin', 0.0); - --- 7. Verification Tokens (API Keys) -INSERT INTO "LiteLLM_VerificationToken" ( - "token", "key_name", "key_alias", "user_id", "team_id", - "models", "spend", "max_budget", "expires", "metadata" -) VALUES - ('e2e-key-update-limits', 'sk-e2e-update', 'e2eUpdateLimitsKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-delete', 'sk-e2e-delete', 'e2eDeleteKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-regenerate', 'sk-e2e-regen', 'e2eRegenerateKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-internal-user', 'sk-e2e-internal', 'e2eInternalUserKey', 'e2e-internal-user', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb), - ('e2e-key-viewer', 'sk-e2e-viewer', 'e2eViewerKey', 'e2e-internal-viewer', NULL, '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb); diff --git a/tests/e2e/ui/globalSetup.ts b/tests/e2e/ui/globalSetup.ts index ef892870268..bab2257a14c 100644 --- a/tests/e2e/ui/globalSetup.ts +++ b/tests/e2e/ui/globalSetup.ts @@ -1,18 +1,22 @@ import { chromium, expect, request } from "@playwright/test"; import { users, Role, STORAGE_PATHS } from "./fixtures/users"; +import { seedGateway } from "./fixtures/apiSeed"; +import { PROXY_BASE_URL } from "./constants"; import * as fs from "fs"; async function globalSetup() { const browser = await chromium.launch(); const rootPath = process.env.SERVER_ROOT_PATH ?? ""; + const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234"; + + await seedGateway(`${PROXY_BASE_URL}${rootPath}`, masterKey); // The Projects sidebar item is hidden unless the enterprise-gated // enable_projects_ui setting is on, and the seeded DB starts with it off. // The proxy runs with LITELLM_LICENSE in CI, so enable it the same way // the admin UI toggle does; the projects migration smoke needs the link. - const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234"; const api = await request.newContext(); - const settingsRes = await api.patch(`http://localhost:4000${rootPath}/update/ui_settings`, { + const settingsRes = await api.patch(`${PROXY_BASE_URL}${rootPath}/update/ui_settings`, { headers: { Authorization: `Bearer ${masterKey}` }, data: { enable_projects_ui: true }, }); @@ -26,7 +30,7 @@ async function globalSetup() { const storagePath = STORAGE_PATHS[role]; const page = await browser.newPage(); try { - await page.goto(`http://localhost:4000${rootPath}/ui/login`); + await page.goto(`${PROXY_BASE_URL}${rootPath}/ui/login`); await page.getByPlaceholder("Enter your username").fill(email); await page.getByPlaceholder("Enter your password").fill(password); await page.getByRole("button", { name: "Login", exact: true }).click(); diff --git a/tests/e2e/ui/migration.serverRootPath.config.ts b/tests/e2e/ui/migration.serverRootPath.config.ts index d32f59b16bf..a8224b17a41 100644 --- a/tests/e2e/ui/migration.serverRootPath.config.ts +++ b/tests/e2e/ui/migration.serverRootPath.config.ts @@ -12,10 +12,10 @@ export default defineConfig({ fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, - workers: process.env.CI ? 1 : undefined, + workers: 1, reporter: "list", use: { - baseURL: "http://localhost:4000", + baseURL: process.env.LITELLM_PROXY_URL ?? "http://localhost:4000", trace: "on-first-retry", actionTimeout: 15 * 1000, navigationTimeout: 30 * 1000, diff --git a/tests/e2e/ui/playwright.config.ts b/tests/e2e/ui/playwright.config.ts index 8d586ce9503..1c8ea32ea86 100644 --- a/tests/e2e/ui/playwright.config.ts +++ b/tests/e2e/ui/playwright.config.ts @@ -13,14 +13,16 @@ export default defineConfig({ forbidOnly: !!process.env.CI, /* Retry on CI only */ retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, + /* One worker everywhere: several specs mutate global gateway state + (router_settings, public model groups), so parallel workers race each + other against the single shared gateway. */ + workers: 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: "html", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: "http://localhost:4000", + baseURL: process.env.LITELLM_PROXY_URL ?? "http://localhost:4000", /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: "on-first-retry", diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index 858eb401c8e..1395aa2aad7 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -2,15 +2,22 @@ set -euo pipefail # ================================================================ -# UI E2E Test Runner (Consolidated) -# Starts postgres, seeds DB, starts mock + proxy, runs Playwright. -# All tests target the proxy on port 4000 (which serves both API -# and UI from the built Next.js static export). +# UI E2E Test Runner +# +# Two modes: +# 1. LITELLM_PROXY_URL set: run Playwright against that live gateway +# (the same one the tests/e2e python suites target). globalSetup +# seeds all e2e-* users/teams/keys/models through the management +# API with LITELLM_MASTER_KEY, so the gateway only needs a master +# key and a database (store_model_in_db: true). +# 2. LITELLM_PROXY_URL unset: provision a disposable gateway first +# (dockerized postgres, dashboard built from source, proxy on +# port 4000), then run the suite against it. # # Usage: -# ./run_e2e.sh # Run once -# ./run_e2e.sh --repeat-each=5 # Run each test 5 times -# ./run_e2e.sh --headed # Run with browser visible +# ./run_e2e.sh # provision + run +# LITELLM_PROXY_URL=http://localhost:4000 ./run_e2e.sh # reuse a gateway +# ./run_e2e.sh --repeat-each=5 # extra args go to playwright # # In CI (CI=true), expects: # - PostgreSQL already running on 127.0.0.1:5432 @@ -24,7 +31,6 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" DASHBOARD_DIR="$REPO_ROOT/ui/litellm-dashboard" IS_CI="${CI:-false}" CONTAINER_NAME="litellm-e2e-postgres-$$" -MOCK_PID="" PROXY_PID="" PROXY_LOG="" @@ -36,10 +42,39 @@ if [ "$IS_CI" = "false" ]; then [ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh" fi -# --- Cleanup on exit --- +# Provider keys for the real-model deployments globalSetup registers via +# /model/new; the shared harness .env is the canonical place they live. +if [ -f "$REPO_ROOT/tests/e2e/.env" ]; then + set -a + source "$REPO_ROOT/tests/e2e/.env" + set +a +fi + +run_playwright() { + echo "=== Installing Playwright dependencies ===" + cd "$SCRIPT_DIR" + npm install --silent 2>/dev/null || true + npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium + + echo "=== Running Playwright tests ===" + npx playwright test --config playwright.config.ts "$@" +} + +# --- Mode 1: reuse an already-running gateway --- +if [ -n "${LITELLM_PROXY_URL:-}" ]; then + echo "=== Using existing gateway at $LITELLM_PROXY_URL ===" + export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-1234}" + curl -fs "$LITELLM_PROXY_URL/health/liveliness" >/dev/null || { + echo "Error: no live proxy at $LITELLM_PROXY_URL" + exit 1 + } + run_playwright "$@" + exit $? +fi + +# --- Mode 2: provision a disposable gateway --- cleanup() { echo "Cleaning up..." - [ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true [ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true [ -n "$PROXY_LOG" ] && rm -f "$PROXY_LOG" || true if [ "$IS_CI" = "false" ]; then @@ -56,10 +91,10 @@ done # --- Database setup --- if [ "$IS_CI" = "false" ]; then - for cmd in docker psql; do + for cmd in docker; do command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } done - for port in 4000 5432 8090; do + for port in 4000 5432; do if lsof -ti ":$port" >/dev/null 2>&1; then echo "Error: port $port is in use" exit 1 @@ -79,7 +114,7 @@ if [ "$IS_CI" = "false" ]; then echo "Waiting for PostgreSQL..." for i in $(seq 1 30); do - if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then + if docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then break fi sleep 1 @@ -91,7 +126,6 @@ fi # --- Credentials --- export LITELLM_MASTER_KEY="sk-1234" -export MOCK_LLM_URL="http://127.0.0.1:8090/v1" export DISABLE_SCHEMA_UPDATE="true" # Ensure the proxy serves UI at /ui (not behind a subpath) export SERVER_ROOT_PATH="" @@ -133,16 +167,6 @@ uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma echo "=== Pushing Prisma schema to database ===" uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss -# --- Mock LLM server --- -echo "=== Starting mock LLM server ===" -uv run --no-sync python "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" & -MOCK_PID=$! - -for i in $(seq 1 15); do - if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi - sleep 1 -done - # --- LiteLLM proxy --- echo "=== Starting LiteLLM proxy ===" cd "$REPO_ROOT" @@ -174,25 +198,5 @@ if [ "$PROXY_READY" -ne 1 ]; then fi echo "Proxy is ready." -# --- Seed database --- -echo "=== Seeding database ===" -DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p') -DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p') -DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p') -DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p') -DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p') - -PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ - -f "$SCRIPT_DIR/fixtures/seed.sql" - -# --- Playwright --- -echo "=== Installing Playwright dependencies ===" -cd "$SCRIPT_DIR" -npm install --silent 2>/dev/null || true -npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium - -echo "=== Running Playwright tests ===" -npx playwright test --config playwright.config.ts "$@" -EXIT_CODE=$? - -exit $EXIT_CODE +run_playwright "$@" +exit $? diff --git a/tests/e2e/ui/tests/auth/unauthenticatedRedirect.spec.ts b/tests/e2e/ui/tests/auth/unauthenticatedRedirect.spec.ts index 4c6e11800ee..f58d4172033 100644 --- a/tests/e2e/ui/tests/auth/unauthenticatedRedirect.spec.ts +++ b/tests/e2e/ui/tests/auth/unauthenticatedRedirect.spec.ts @@ -1,8 +1,9 @@ import { test, expect } from "@playwright/test"; +import { PROXY_BASE_URL } from "../../constants"; test.describe("Authentication Checks", () => { test("should redirect unauthenticated user from a protected page", async ({ page }) => { - const protectedPageUrl = "http://localhost:4000/ui?page=llm-playground"; + const protectedPageUrl = `${PROXY_BASE_URL}/ui?page=llm-playground`; await page.goto(protectedPageUrl, { waitUntil: "domcontentloaded" }); await expect(page).toHaveURL(/\/ui\/login/); await expect(page.getByRole("heading", { name: "Login" })).toBeVisible(); diff --git a/tests/e2e/ui/tests/login/login.spec.ts b/tests/e2e/ui/tests/login/login.spec.ts index 88378df36c3..068eabba7a0 100644 --- a/tests/e2e/ui/tests/login/login.spec.ts +++ b/tests/e2e/ui/tests/login/login.spec.ts @@ -1,9 +1,10 @@ import { expect, test } from "@playwright/test"; import { users } from "../../fixtures/users"; import { Role } from "../../fixtures/roles"; +import { PROXY_BASE_URL } from "../../constants"; test("user can log in", async ({ page }) => { - await page.goto("http://localhost:4000/ui/login"); + await page.goto(`${PROXY_BASE_URL}/ui/login`); await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email); await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password); const loginButton = page.getByRole("button", { name: "Login", exact: true }); diff --git a/tests/e2e/ui/tests/login/serverRootPathRedirect.spec.ts b/tests/e2e/ui/tests/login/serverRootPathRedirect.spec.ts index 37fea73961f..33138209004 100644 --- a/tests/e2e/ui/tests/login/serverRootPathRedirect.spec.ts +++ b/tests/e2e/ui/tests/login/serverRootPathRedirect.spec.ts @@ -1,4 +1,5 @@ import { expect, test } from "@playwright/test"; +import { PROXY_BASE_URL } from "../../constants"; // Driven by the SERVER_ROOT_PATH env var injected by the workflow; the container // is booted with the same value, so the asset paths and the runtime config it @@ -24,7 +25,7 @@ test("unauth redirect preserves SERVER_ROOT_PATH prefix", async ({ page }) => { await page.context().clearCookies(); - await page.goto(`http://localhost:4000${ROOT_PATH}/ui/?page=virtual-keys`); + await page.goto(`${PROXY_BASE_URL}${ROOT_PATH}/ui/?page=virtual-keys`); await page.waitForURL((url) => url.pathname.includes("/ui/login"), { timeout: 15_000 }); diff --git a/tests/e2e/ui/tests/mcp/mcpServers.spec.ts b/tests/e2e/ui/tests/mcp/mcpServers.spec.ts index 7c4a7cb0568..f3ccd644937 100644 --- a/tests/e2e/ui/tests/mcp/mcpServers.spec.ts +++ b/tests/e2e/ui/tests/mcp/mcpServers.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect, APIRequestContext } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; @@ -8,9 +8,34 @@ import { Page } from "../../fixtures/pages"; // — SSE / stdio / OpenAPI transports, API Key / Bearer / OAuth2 / Basic / Token // / AWS SigV4 auth, edit/delete, BYOK credentials, tool list/call (needs a real // or mocked MCP server in the e2e fixture stack), and access-group permissions. + +// Leftover e2e_mcp_* servers point at a dead URL, and the MCP page's health +// checks against them keep the network busy long enough that navigation's +// networkidle wait times out on the next run against a persistent gateway. +async function purgeE2eMcpServers(request: APIRequestContext): Promise { + const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234"; + const auth = { Authorization: `Bearer ${masterKey}` }; + const res = await request.get("/v1/mcp/server", { headers: auth }); + if (!res.ok()) return; + const servers: Array<{ server_id: string; server_name?: string }> = await res.json(); + for (const server of servers) { + if (server.server_name?.startsWith("e2e_mcp_")) { + await request.delete(`/v1/mcp/server/${server.server_id}`, { headers: auth }); + } + } +} + test.describe("MCP Servers", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); + test.beforeEach(async ({ request }) => { + await purgeE2eMcpServers(request); + }); + + test.afterEach(async ({ request }) => { + await purgeE2eMcpServers(request); + }); + test("Add a custom MCP server via the discovery → custom form", async ({ page }) => { await navigateToPage(page, Page.McpServers); diff --git a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts index ca9c35ce722..e30cb73ad7f 100644 --- a/tests/e2e/ui/tests/modelHub/modelHub.spec.ts +++ b/tests/e2e/ui/tests/modelHub/modelHub.spec.ts @@ -6,6 +6,18 @@ import { Page } from "../../fixtures/pages"; test.describe("AI Hub (internal admin view)", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); + // Reset public model groups so the run is idempotent against a persistent + // gateway: make_public replaces the whole list, and once a prior run made + // every group public the modal's Select All has zero rows left to pick. + test.beforeEach(async ({ page }) => { + const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234"; + const res = await page.request.post("/model_group/make_public", { + headers: { Authorization: `Bearer ${masterKey}` }, + data: { model_groups: [] }, + }); + expect(res.ok(), `resetting public model groups failed: ${res.status()}`).toBe(true); + }); + test("Make models public via the multi-step modal", async ({ page }) => { await navigateToPage(page, Page.ModelHubTable); diff --git a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts index c44957ea737..d2ce551fbc9 100644 --- a/tests/e2e/ui/tests/proxy-admin/keys.spec.ts +++ b/tests/e2e/ui/tests/proxy-admin/keys.spec.ts @@ -6,6 +6,7 @@ import { E2E_UPDATE_LIMITS_KEY_ALIAS, E2E_INTERNAL_USER_KEY_ALIAS, E2E_TEAM_CRUD_ALIAS, + E2E_UI_OPENAI_MODEL, } from "../../constants"; import { Page } from "../../fixtures/pages"; import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; @@ -166,7 +167,7 @@ test.describe("Proxy Admin - Keys", () => { // Open the model multi-select and pick a single specific model. Use // getByRole("option", ...) to avoid the strict-mode collision between // the option container and its inner text node. - const modelName = "fake-openai-gpt-4"; + const modelName = E2E_UI_OPENAI_MODEL; await page.locator(".ant-select-selection-overflow").click(); const option = page.locator(".ant-select-dropdown:visible").getByRole("option", { name: modelName, exact: true }); await option.waitFor({ state: "attached" }); @@ -182,8 +183,8 @@ test.describe("Proxy Admin - Keys", () => { // Grab the new key from the success modal (rendered inside a
) and
     // verify it can call /chat/completions for the model it was scoped to.
-    // The mock LLM server (fixtures/mock_llm_server/server.py) replies with
-    // a fixed "This is a mock response." body.
+    // The deployment routes to a real provider, so this costs a fraction of
+    // a cent and proves the whole key -> router -> provider path.
     const apiKey = (await page.locator(".ant-modal:visible pre").innerText()).trim();
     expect(apiKey).toMatch(/^sk-/);
 
@@ -191,12 +192,12 @@ test.describe("Proxy Admin - Keys", () => {
       headers: { Authorization: `Bearer ${apiKey}` },
       data: {
         model: modelName,
-        messages: [{ role: "user", content: "ping" }],
+        messages: [{ role: "user", content: "Reply with the single word: pong" }],
       },
     });
     expect(response.status()).toBe(200);
     const body = await response.json();
-    expect(body.choices?.[0]?.message?.content).toBe("This is a mock response.");
+    expect(body.choices?.[0]?.message?.content?.length).toBeGreaterThan(0);
 
     await page.keyboard.press("Escape");
 
diff --git a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts
index e7f67d7367f..e1b9c2d657b 100644
--- a/tests/e2e/ui/tests/proxy-admin/teams.spec.ts
+++ b/tests/e2e/ui/tests/proxy-admin/teams.spec.ts
@@ -5,6 +5,9 @@ import {
   E2E_TEAM_DELETE_ALIAS,
   E2E_TEAM_NO_ADMIN_ID,
   E2E_TEAM_ORG_ID,
+  E2E_UI_ANTHROPIC_MODEL,
+  E2E_UI_OPENAI_MODEL,
+  PROXY_BASE_URL,
 } from "../../constants";
 import { Page } from "../../fixtures/pages";
 import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
@@ -127,12 +130,12 @@ test.describe("Proxy Admin - Teams", () => {
 
   test("Edit team model selection", async ({ page, request }) => {
     // Restore the seeded models via API in case a prior run (or a CI retry)
-    // left this team mutated — the assertion below requires fake-anthropic-claude
-    // to be present.
+    // left this team mutated — the assertion below requires the anthropic
+    // deployment to be present.
     const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234";
-    const seededModels = ["fake-openai-gpt-4", "fake-anthropic-claude"];
+    const seededModels = [E2E_UI_OPENAI_MODEL, E2E_UI_ANTHROPIC_MODEL];
     const restore = async () => {
-      const res = await request.post("http://localhost:4000/team/update", {
+      const res = await request.post(`${PROXY_BASE_URL}/team/update`, {
         headers: { Authorization: `Bearer ${masterKey}` },
         data: { team_id: E2E_TEAM_CRUD_ID, models: seededModels },
       });
@@ -156,7 +159,7 @@ test.describe("Proxy Admin - Teams", () => {
 
       const anthropicTag = modelsSelect
         .locator(".ant-select-selection-item")
-        .filter({ hasText: "fake-anthropic-claude" });
+        .filter({ hasText: E2E_UI_ANTHROPIC_MODEL });
       await expect(anthropicTag).toBeVisible({ timeout: 5_000 });
       await anthropicTag.locator(".ant-select-selection-item-remove").click();
 
diff --git a/tests/e2e/ui/tests/settings/routerSettings.spec.ts b/tests/e2e/ui/tests/settings/routerSettings.spec.ts
index ffa5f2c2ae2..b7c087a43af 100644
--- a/tests/e2e/ui/tests/settings/routerSettings.spec.ts
+++ b/tests/e2e/ui/tests/settings/routerSettings.spec.ts
@@ -1,5 +1,5 @@
 import { test, expect } from "@playwright/test";
-import { ADMIN_STORAGE_PATH } from "../../constants";
+import { ADMIN_STORAGE_PATH, E2E_UI_ANTHROPIC_MODEL, E2E_UI_OPENAI_MODEL, PROXY_BASE_URL } from "../../constants";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
 import { Role, users } from "../../fixtures/users";
@@ -12,8 +12,8 @@ import type { components } from "../../../../../ui/litellm-dashboard/src/lib/htt
 // echoes the whole settings object, so they must not run concurrently.
 test.describe.configure({ mode: "serial" });
 
-const PRIMARY = "fake-openai-gpt-4";
-const FALLBACK = "fake-anthropic-claude";
+const PRIMARY = E2E_UI_OPENAI_MODEL;
+const FALLBACK = E2E_UI_ANTHROPIC_MODEL;
 
 /**
  * Wipe any fallbacks for the primary model so the test is idempotent across
@@ -23,7 +23,7 @@ async function clearFallbackForPrimary(request: import("@playwright/test").APIRe
   const masterKey = users[Role.ProxyAdmin].password;
   const auth = { Authorization: `Bearer ${masterKey}` };
 
-  const current = await request.get("http://localhost:4000/get/config/callbacks", { headers: auth });
+  const current = await request.get(`${PROXY_BASE_URL}/get/config/callbacks`, { headers: auth });
   if (!current.ok()) return;
   const body = await current.json();
   const router = body?.router_settings ?? {};
@@ -31,7 +31,7 @@ async function clearFallbackForPrimary(request: import("@playwright/test").APIRe
   const next = existing.filter((entry) => !(entry && PRIMARY in entry));
   if (next.length === existing.length) return;
 
-  await request.post("http://localhost:4000/config/update", {
+  await request.post(`${PROXY_BASE_URL}/config/update`, {
     headers: auth,
     data: { router_settings: { ...router, fallbacks: next } },
   });
@@ -111,7 +111,7 @@ test.describe("Router Settings - Fallbacks", () => {
 type ConfigYAML = components["schemas"]["ConfigYAML"];
 type RouterSettingsResponse = components["schemas"]["RouterSettingsResponse"];
 
-const BASE_URL = "http://localhost:4000";
+const BASE_URL = PROXY_BASE_URL;
 const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` };
 
 /**