mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
test(e2e): run the UI Playwright suite against the shared e2e gateway
The suite now targets LITELLM_PROXY_URL (default http://localhost:4000), the same live gateway the python e2e suites use, instead of provisioning its own postgres, mock LLM server and SQL-seeded proxy. globalSetup seeds every e2e-* user, team, org, key and model deployment through the management API with the master key, so any gateway with a master key and store_model_in_db works; seed.sql and the mock LLM server are gone. The two deployments (e2e-ui-openai, e2e-ui-anthropic) route to real providers per the harness's no-mocks rule, and the key creation spec now asserts a real completion instead of the mock's canned body. Seeding is idempotent against a persistent gateway: teardown deletes per-entity (batch deletes are all-or-nothing and one missing id used to strand every other row), the modelHub spec resets the public model groups list it fills, and the MCP spec purges its dead-URL servers whose failing health checks kept networkidle navigation from settling on reruns. Both Playwright configs pin workers: 1 since several specs mutate global gateway state; the suite documents that it needs the gateway to itself while it runs. Two specs are also caught up with UI refactors that had moved their buttons into dropdown menus (key deletion under "More key actions", credential editing under the row's actions menu). run_e2e.sh keeps a provisioning mode for CI and machines with no gateway, and skips provisioning entirely when LITELLM_PROXY_URL is set. The CircleCI jobs drop their seed and mock-server steps; the mock server's removal also lets tests/e2e/ui leave the basedpyright exclude list.
This commit is contained in:
ryan-crabbe-berri
parent
f1b567f1bc
commit
871629e8f3
20 changed files with 268 additions and 317 deletions
|
|
@ -2776,20 +2776,10 @@ jobs:
|
|||
- run:
|
||||
name: Push Prisma schema
|
||||
command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
||||
- run:
|
||||
name: Seed database
|
||||
command: |
|
||||
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
||||
-f tests/e2e/ui/fixtures/seed.sql
|
||||
- run:
|
||||
name: Start mock LLM server
|
||||
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
|
||||
background: true
|
||||
- run:
|
||||
name: Start LiteLLM proxy
|
||||
environment:
|
||||
LITELLM_MASTER_KEY: "sk-1234"
|
||||
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
||||
DISABLE_SCHEMA_UPDATE: "true"
|
||||
SERVER_ROOT_PATH: ""
|
||||
# PROXY_LOGOUT_URL is inherited from the job-level environment so the
|
||||
|
|
@ -2904,20 +2894,10 @@ jobs:
|
|||
- run:
|
||||
name: Push Prisma schema
|
||||
command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
||||
- run:
|
||||
name: Seed database
|
||||
command: |
|
||||
PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
|
||||
-f tests/e2e/ui/fixtures/seed.sql
|
||||
- run:
|
||||
name: Start mock LLM server
|
||||
command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py
|
||||
background: true
|
||||
- run:
|
||||
name: Start LiteLLM proxy under a server root path
|
||||
environment:
|
||||
LITELLM_MASTER_KEY: "sk-1234"
|
||||
MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
|
||||
DISABLE_SCHEMA_UPDATE: "true"
|
||||
# Output flows to this step's own log, so a boot crash is visible here
|
||||
# rather than swallowed by a downstream readiness probe.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"include": ["litellm"],
|
||||
"ignore": [],
|
||||
"exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "tests/e2e/ui", "litellm/types/utils.py", "litellm/proxy/_types.py"],
|
||||
"exclude": ["**/node_modules", "**/__pycache__", "tests/e2e/claude_code", "litellm/types/utils.py", "litellm/proxy/_types.py"],
|
||||
"pythonVersion": "3.12",
|
||||
"typeCheckingMode": "strict",
|
||||
"enableTypeIgnoreComments": false,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
|
|||
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
|
||||
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
|
||||
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
|
||||
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`
|
||||
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard at `LITELLM_PROXY_URL` (default `http://localhost:4000`), the same live gateway the python suites target. The gateway only needs a master key and a database (`store_model_in_db: true`); `globalSetup` provisions everything else (e2e-* users, teams, keys, and real-provider model deployments) through the management API and `run_e2e.sh` can stand up a disposable gateway when none is running. The suite runs single-worker and needs the gateway to itself while it runs: several specs mutate global gateway state (`router_settings` via `/config/update`, the public model groups list, `enable_projects_ui`), so do not run the python suites against the same gateway at the same time. It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it
|
||||
|
||||
## MCP suite: real Datadog only
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
export const PROXY_BASE_URL = (process.env.LITELLM_PROXY_URL ?? "http://localhost:4000").replace(/\/+$/, "");
|
||||
|
||||
export const E2E_UI_OPENAI_MODEL = "e2e-ui-openai";
|
||||
export const E2E_UI_ANTHROPIC_MODEL = "e2e-ui-anthropic";
|
||||
|
||||
// Storage state paths for each role
|
||||
export const ADMIN_STORAGE_PATH = "admin.storageState.json";
|
||||
export const ADMIN_VIEWER_STORAGE_PATH = "adminViewer.storageState.json";
|
||||
|
|
@ -5,20 +10,20 @@ export const INTERNAL_USER_STORAGE_PATH = "internalUser.storageState.json";
|
|||
export const INTERNAL_VIEWER_STORAGE_PATH = "internalViewer.storageState.json";
|
||||
export const TEAM_ADMIN_STORAGE_PATH = "teamAdmin.storageState.json";
|
||||
|
||||
// Seeded user identities (match seed.sql)
|
||||
// Seeded user identities (match fixtures/apiSeed.ts)
|
||||
export const E2E_PROXY_ADMIN_USER_ID = "e2e-proxy-admin";
|
||||
export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local";
|
||||
export const E2E_INTERNAL_USER_ID = "e2e-internal-user";
|
||||
export const E2E_INTERNAL_USER_EMAIL = "internal@test.local";
|
||||
|
||||
// Key aliases for seeded test keys (match seed.sql)
|
||||
// Key aliases for seeded test keys (match fixtures/apiSeed.ts)
|
||||
export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey";
|
||||
export const E2E_DELETE_KEY_ALIAS = "e2eDeleteKey";
|
||||
export const E2E_REGENERATE_KEY_ALIAS = "e2eRegenerateKey";
|
||||
export const E2E_INTERNAL_USER_KEY_ALIAS = "e2eInternalUserKey";
|
||||
export const E2E_VIEWER_KEY_ALIAS = "e2eViewerKey";
|
||||
|
||||
// Team identifiers (match seed.sql)
|
||||
// Team identifiers (match fixtures/apiSeed.ts)
|
||||
export const E2E_TEAM_CRUD_ID = "e2e-team-crud";
|
||||
export const E2E_TEAM_CRUD_ALIAS = "E2E Team CRUD";
|
||||
export const E2E_TEAM_DELETE_ID = "e2e-team-delete";
|
||||
|
|
|
|||
130
tests/e2e/ui/fixtures/apiSeed.ts
Normal file
130
tests/e2e/ui/fixtures/apiSeed.ts
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
import { APIRequestContext, request } from "@playwright/test";
|
||||
import { E2E_UI_OPENAI_MODEL, E2E_UI_ANTHROPIC_MODEL } from "../constants";
|
||||
|
||||
const OPENAI_UPSTREAM = `openai/${process.env.E2E_CHEAP_OPENAI_MODEL ?? "gpt-5.5"}`;
|
||||
const ANTHROPIC_UPSTREAM = `anthropic/${process.env.E2E_CHEAP_ANTHROPIC_MODEL ?? "claude-haiku-4-5"}`;
|
||||
|
||||
const SEED_PASSWORD = "test";
|
||||
|
||||
const MODELS = [
|
||||
{
|
||||
model_name: E2E_UI_OPENAI_MODEL,
|
||||
litellm_params: { model: OPENAI_UPSTREAM, api_key: "os.environ/OPENAI_API_KEY" },
|
||||
model_info: { id: E2E_UI_OPENAI_MODEL },
|
||||
},
|
||||
{
|
||||
model_name: E2E_UI_ANTHROPIC_MODEL,
|
||||
litellm_params: { model: ANTHROPIC_UPSTREAM, api_key: "os.environ/ANTHROPIC_API_KEY" },
|
||||
model_info: { id: E2E_UI_ANTHROPIC_MODEL },
|
||||
},
|
||||
] as const;
|
||||
|
||||
const ORG = { organization_id: "e2e-org-main", organization_alias: "E2E Organization", max_budget: 1000 } as const;
|
||||
|
||||
const USERS = [
|
||||
{ user_id: "e2e-proxy-admin", user_email: "admin@test.local", user_role: "proxy_admin" },
|
||||
{ user_id: "e2e-admin-viewer", user_email: "adminviewer@test.local", user_role: "proxy_admin_viewer" },
|
||||
{ user_id: "e2e-internal-user", user_email: "internal@test.local", user_role: "internal_user" },
|
||||
{ user_id: "e2e-internal-viewer", user_email: "viewer@test.local", user_role: "internal_user_viewer" },
|
||||
{ user_id: "e2e-team-admin", user_email: "teamadmin@test.local", user_role: "internal_user" },
|
||||
{ user_id: "e2e-invitable-user", user_email: "invitable@test.local", user_role: "internal_user" },
|
||||
{ user_id: "e2e-internal-noteam", user_email: "noteam@test.local", user_role: "internal_user" },
|
||||
{ user_id: "e2e-invitable-by-team-admin", user_email: "invitable-team@test.local", user_role: "internal_user" },
|
||||
{ user_id: "e2e-removable-member", user_email: "removable@test.local", user_role: "internal_user" },
|
||||
] as const;
|
||||
|
||||
const TEAMS = [
|
||||
{
|
||||
team_id: "e2e-team-crud",
|
||||
team_alias: "E2E Team CRUD",
|
||||
organization_id: null,
|
||||
models: [E2E_UI_OPENAI_MODEL, E2E_UI_ANTHROPIC_MODEL],
|
||||
members_with_roles: [
|
||||
{ role: "admin", user_id: "e2e-team-admin" },
|
||||
{ role: "user", user_id: "e2e-internal-user" },
|
||||
{ role: "user", user_id: "e2e-internal-viewer" },
|
||||
{ role: "user", user_id: "e2e-removable-member" },
|
||||
],
|
||||
},
|
||||
{
|
||||
team_id: "e2e-team-delete",
|
||||
team_alias: "E2E Team Delete",
|
||||
organization_id: null,
|
||||
models: [E2E_UI_OPENAI_MODEL],
|
||||
members_with_roles: [{ role: "admin", user_id: "e2e-team-admin" }],
|
||||
},
|
||||
{
|
||||
team_id: "e2e-team-org",
|
||||
team_alias: "E2E Team In Org",
|
||||
organization_id: ORG.organization_id,
|
||||
models: [E2E_UI_OPENAI_MODEL],
|
||||
members_with_roles: [{ role: "user", user_id: "e2e-internal-user" }],
|
||||
},
|
||||
{
|
||||
team_id: "e2e-team-no-admin",
|
||||
team_alias: "E2E Team No Admin",
|
||||
organization_id: null,
|
||||
models: [E2E_UI_OPENAI_MODEL],
|
||||
members_with_roles: [{ role: "user", user_id: "e2e-invitable-user" }],
|
||||
},
|
||||
] as const;
|
||||
|
||||
const KEYS = [
|
||||
{ key_alias: "e2eUpdateLimitsKey", user_id: "e2e-proxy-admin", team_id: "e2e-team-crud" },
|
||||
{ key_alias: "e2eDeleteKey", user_id: "e2e-proxy-admin", team_id: "e2e-team-crud" },
|
||||
{ key_alias: "e2eRegenerateKey", user_id: "e2e-proxy-admin", team_id: "e2e-team-crud" },
|
||||
{ key_alias: "e2eInternalUserKey", user_id: "e2e-internal-user", team_id: "e2e-team-crud" },
|
||||
{ key_alias: "e2eViewerKey", user_id: "e2e-internal-viewer", team_id: null },
|
||||
] as const;
|
||||
|
||||
async function post(api: APIRequestContext, path: string, data: unknown, allowFailure = false): Promise<void> {
|
||||
const res = await api.post(path, { data });
|
||||
if (res.ok() || allowFailure) return;
|
||||
throw new Error(`Seeding ${path} failed (${res.status()}): ${await res.text()}`);
|
||||
}
|
||||
|
||||
async function teardown(api: APIRequestContext): Promise<void> {
|
||||
for (const key of KEYS) {
|
||||
await post(api, "/key/delete", { key_aliases: [key.key_alias] }, true);
|
||||
}
|
||||
for (const team of TEAMS) {
|
||||
await post(api, "/team/delete", { team_ids: [team.team_id] }, true);
|
||||
}
|
||||
for (const user of USERS) {
|
||||
await post(api, "/user/delete", { user_ids: [user.user_id] }, true);
|
||||
}
|
||||
await api.delete("/organization/delete", { data: { organization_ids: [ORG.organization_id] } });
|
||||
for (const model of MODELS) {
|
||||
await post(api, "/model/delete", { id: model.model_info.id }, true);
|
||||
}
|
||||
}
|
||||
|
||||
async function create(api: APIRequestContext): Promise<void> {
|
||||
for (const model of MODELS) {
|
||||
await post(api, "/model/new", model);
|
||||
}
|
||||
await post(api, "/organization/new", ORG);
|
||||
for (const user of USERS) {
|
||||
await post(api, "/user/new", { ...user, auto_create_key: false });
|
||||
await post(api, "/user/update", { user_id: user.user_id, password: SEED_PASSWORD });
|
||||
}
|
||||
for (const team of TEAMS) {
|
||||
await post(api, "/team/new", team);
|
||||
}
|
||||
for (const key of KEYS) {
|
||||
await post(api, "/key/generate", { ...key, models: [E2E_UI_OPENAI_MODEL] });
|
||||
}
|
||||
}
|
||||
|
||||
export async function seedGateway(baseUrl: string, masterKey: string): Promise<void> {
|
||||
const api = await request.newContext({
|
||||
baseURL: baseUrl,
|
||||
extraHTTPHeaders: { Authorization: `Bearer ${masterKey}` },
|
||||
});
|
||||
try {
|
||||
await teardown(api);
|
||||
await create(api);
|
||||
} finally {
|
||||
await api.dispose();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +1,3 @@
|
|||
model_list:
|
||||
- model_name: fake-openai-gpt-4
|
||||
litellm_params:
|
||||
model: openai/fake-gpt-4
|
||||
api_base: os.environ/MOCK_LLM_URL
|
||||
api_key: fake-key
|
||||
- model_name: fake-anthropic-claude
|
||||
litellm_params:
|
||||
model: openai/fake-claude
|
||||
api_base: os.environ/MOCK_LLM_URL
|
||||
api_key: fake-key
|
||||
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
database_url: os.environ/DATABASE_URL
|
||||
|
|
|
|||
|
|
@ -1,120 +0,0 @@
|
|||
"""
|
||||
Mock LLM server for UI e2e tests.
|
||||
Responds to OpenAI-format endpoints with canned responses.
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
|
||||
app = FastAPI(title="Mock LLM Server")
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/v1/models")
|
||||
@app.get("/models")
|
||||
async def list_models():
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": "fake-gpt-4", "object": "model", "owned_by": "mock"},
|
||||
{"id": "fake-claude", "object": "model", "owned_by": "mock"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
@app.post("/chat/completions")
|
||||
async def chat_completions(request: Request):
|
||||
body = await request.json()
|
||||
model = body.get("model", "mock-model")
|
||||
stream = body.get("stream", False)
|
||||
|
||||
response_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
|
||||
created = int(time.time())
|
||||
|
||||
if stream:
|
||||
|
||||
async def stream_generator():
|
||||
chunk = {
|
||||
"id": response_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": created,
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"role": "assistant",
|
||||
"content": "This is a mock response.",
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
yield f"data: {json.dumps(chunk)}\n\n"
|
||||
|
||||
done_chunk = {
|
||||
"id": response_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": created,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
}
|
||||
yield f"data: {json.dumps(done_chunk)}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
return StreamingResponse(stream_generator(), media_type="text/event-stream")
|
||||
|
||||
return {
|
||||
"id": response_id,
|
||||
"object": "chat.completion",
|
||||
"created": created,
|
||||
"model": model,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "This is a mock response."},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
|
||||
}
|
||||
|
||||
|
||||
@app.post("/v1/embeddings")
|
||||
@app.post("/embeddings")
|
||||
async def embeddings(request: Request):
|
||||
body = await request.json()
|
||||
inputs = body.get("input", [""])
|
||||
if isinstance(inputs, str):
|
||||
inputs = [inputs]
|
||||
return {
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"object": "embedding", "index": i, "embedding": [0.0] * 1536}
|
||||
for i in range(len(inputs))
|
||||
],
|
||||
"model": body.get("model", "mock-embedding"),
|
||||
"usage": {"prompt_tokens": 5, "total_tokens": 5},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run(app, host="127.0.0.1", port=8090)
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
-- E2E Test Seed Data
|
||||
-- Idempotent: deletes all e2e-* rows then re-inserts deterministic data.
|
||||
|
||||
-- 1. Clean up in dependency order
|
||||
DELETE FROM "LiteLLM_TeamMembership" WHERE "user_id" LIKE 'e2e-%';
|
||||
DELETE FROM "LiteLLM_VerificationToken" WHERE token LIKE 'e2e-%';
|
||||
DELETE FROM "LiteLLM_TeamTable" WHERE "team_id" LIKE 'e2e-%';
|
||||
DELETE FROM "LiteLLM_OrganizationTable" WHERE "organization_id" LIKE 'e2e-%';
|
||||
DELETE FROM "LiteLLM_UserTable" WHERE "user_id" LIKE 'e2e-%';
|
||||
DELETE FROM "LiteLLM_BudgetTable" WHERE "budget_id" LIKE 'e2e-%';
|
||||
|
||||
-- 2. Budget (created_by and updated_by are NOT NULL)
|
||||
INSERT INTO "LiteLLM_BudgetTable" ("budget_id", "max_budget", "created_by", "updated_by")
|
||||
VALUES ('e2e-budget-org', 1000, 'e2e-proxy-admin', 'e2e-proxy-admin');
|
||||
|
||||
-- 3. Organization (created_by and updated_by are NOT NULL)
|
||||
INSERT INTO "LiteLLM_OrganizationTable" (
|
||||
"organization_id", "organization_alias", "budget_id",
|
||||
"metadata", "models", "spend", "model_spend",
|
||||
"created_by", "updated_by"
|
||||
) VALUES (
|
||||
'e2e-org-main', 'E2E Organization', 'e2e-budget-org',
|
||||
'{}'::jsonb, ARRAY[]::text[], 0.0, '{}'::jsonb,
|
||||
'e2e-proxy-admin', 'e2e-proxy-admin'
|
||||
);
|
||||
|
||||
-- 4. Users (password hash is scrypt of "test")
|
||||
INSERT INTO "LiteLLM_UserTable" ("user_id", "user_email", "user_role", "teams", "password")
|
||||
VALUES
|
||||
('e2e-proxy-admin', 'admin@test.local', 'proxy_admin', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-admin-viewer', 'adminviewer@test.local', 'proxy_admin_viewer', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-internal-user', 'internal@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-org"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-internal-viewer', 'viewer@test.local', 'internal_user_viewer', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-team-admin', 'teamadmin@test.local', 'internal_user', '{"e2e-team-crud","e2e-team-delete"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-invitable-user', 'invitable@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-internal-noteam', 'noteam@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-invitable-by-team-admin', 'invitable-team@test.local', 'internal_user', '{}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr'),
|
||||
('e2e-removable-member', 'removable@test.local', 'internal_user', '{"e2e-team-crud"}', 'scrypt:MU5CcTAi6rVK1HfY1rVPEWq6r4sxg837eq9dG4n5Q6BhDJ44442+seC6LAhLEAYr');
|
||||
|
||||
-- 5. Teams (members_with_roles is required JSON)
|
||||
INSERT INTO "LiteLLM_TeamTable" (
|
||||
"team_id", "team_alias", "organization_id", "admins", "members",
|
||||
"members_with_roles", "metadata", "models", "spend", "model_spend", "model_max_budget", "blocked"
|
||||
) VALUES
|
||||
('e2e-team-crud', 'E2E Team CRUD', NULL,
|
||||
'{"e2e-team-admin"}',
|
||||
'{"e2e-team-admin","e2e-internal-user","e2e-internal-viewer","e2e-removable-member"}',
|
||||
'[{"role":"admin","user_id":"e2e-team-admin"},{"role":"user","user_id":"e2e-internal-user"},{"role":"user","user_id":"e2e-internal-viewer"},{"role":"user","user_id":"e2e-removable-member"}]'::jsonb,
|
||||
'{}'::jsonb, '{"fake-openai-gpt-4","fake-anthropic-claude"}', 0.0, '{}'::jsonb, '{}'::jsonb, false),
|
||||
|
||||
('e2e-team-delete', 'E2E Team Delete', NULL,
|
||||
'{"e2e-team-admin"}', '{"e2e-team-admin"}',
|
||||
'[{"role":"admin","user_id":"e2e-team-admin"}]'::jsonb,
|
||||
'{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false),
|
||||
|
||||
('e2e-team-org', 'E2E Team In Org', 'e2e-org-main',
|
||||
'{}', '{"e2e-internal-user"}',
|
||||
'[{"role":"user","user_id":"e2e-internal-user"}]'::jsonb,
|
||||
'{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false),
|
||||
|
||||
('e2e-team-no-admin', 'E2E Team No Admin', NULL,
|
||||
'{}', '{"e2e-invitable-user"}',
|
||||
'[{"role":"user","user_id":"e2e-invitable-user"}]'::jsonb,
|
||||
'{}'::jsonb, '{"fake-openai-gpt-4"}', 0.0, '{}'::jsonb, '{}'::jsonb, false);
|
||||
|
||||
-- 6. Team Memberships (only user_id, team_id, spend — no created_at/updated_at)
|
||||
INSERT INTO "LiteLLM_TeamMembership" ("user_id", "team_id", "spend")
|
||||
VALUES
|
||||
('e2e-team-admin', 'e2e-team-crud', 0.0),
|
||||
('e2e-internal-user', 'e2e-team-crud', 0.0),
|
||||
('e2e-internal-viewer', 'e2e-team-crud', 0.0),
|
||||
('e2e-removable-member', 'e2e-team-crud', 0.0),
|
||||
('e2e-team-admin', 'e2e-team-delete', 0.0),
|
||||
('e2e-internal-user', 'e2e-team-org', 0.0),
|
||||
('e2e-invitable-user', 'e2e-team-no-admin', 0.0);
|
||||
|
||||
-- 7. Verification Tokens (API Keys)
|
||||
INSERT INTO "LiteLLM_VerificationToken" (
|
||||
"token", "key_name", "key_alias", "user_id", "team_id",
|
||||
"models", "spend", "max_budget", "expires", "metadata"
|
||||
) VALUES
|
||||
('e2e-key-update-limits', 'sk-e2e-update', 'e2eUpdateLimitsKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
|
||||
('e2e-key-delete', 'sk-e2e-delete', 'e2eDeleteKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
|
||||
('e2e-key-regenerate', 'sk-e2e-regen', 'e2eRegenerateKey', 'e2e-proxy-admin', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
|
||||
('e2e-key-internal-user', 'sk-e2e-internal', 'e2eInternalUserKey', 'e2e-internal-user', 'e2e-team-crud', '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb),
|
||||
('e2e-key-viewer', 'sk-e2e-viewer', 'e2eViewerKey', 'e2e-internal-viewer', NULL, '{"fake-openai-gpt-4"}', 0.0, NULL, NULL, '{}'::jsonb);
|
||||
|
|
@ -1,18 +1,22 @@
|
|||
import { chromium, expect, request } from "@playwright/test";
|
||||
import { users, Role, STORAGE_PATHS } from "./fixtures/users";
|
||||
import { seedGateway } from "./fixtures/apiSeed";
|
||||
import { PROXY_BASE_URL } from "./constants";
|
||||
import * as fs from "fs";
|
||||
|
||||
async function globalSetup() {
|
||||
const browser = await chromium.launch();
|
||||
const rootPath = process.env.SERVER_ROOT_PATH ?? "";
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234";
|
||||
|
||||
await seedGateway(`${PROXY_BASE_URL}${rootPath}`, masterKey);
|
||||
|
||||
// The Projects sidebar item is hidden unless the enterprise-gated
|
||||
// enable_projects_ui setting is on, and the seeded DB starts with it off.
|
||||
// The proxy runs with LITELLM_LICENSE in CI, so enable it the same way
|
||||
// the admin UI toggle does; the projects migration smoke needs the link.
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234";
|
||||
const api = await request.newContext();
|
||||
const settingsRes = await api.patch(`http://localhost:4000${rootPath}/update/ui_settings`, {
|
||||
const settingsRes = await api.patch(`${PROXY_BASE_URL}${rootPath}/update/ui_settings`, {
|
||||
headers: { Authorization: `Bearer ${masterKey}` },
|
||||
data: { enable_projects_ui: true },
|
||||
});
|
||||
|
|
@ -26,7 +30,7 @@ async function globalSetup() {
|
|||
const storagePath = STORAGE_PATHS[role];
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await page.goto(`http://localhost:4000${rootPath}/ui/login`);
|
||||
await page.goto(`${PROXY_BASE_URL}${rootPath}/ui/login`);
|
||||
await page.getByPlaceholder("Enter your username").fill(email);
|
||||
await page.getByPlaceholder("Enter your password").fill(password);
|
||||
await page.getByRole("button", { name: "Login", exact: true }).click();
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ export default defineConfig({
|
|||
fullyParallel: true,
|
||||
forbidOnly: !!process.env.CI,
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
workers: process.env.CI ? 1 : undefined,
|
||||
workers: 1,
|
||||
reporter: "list",
|
||||
use: {
|
||||
baseURL: "http://localhost:4000",
|
||||
baseURL: process.env.LITELLM_PROXY_URL ?? "http://localhost:4000",
|
||||
trace: "on-first-retry",
|
||||
actionTimeout: 15 * 1000,
|
||||
navigationTimeout: 30 * 1000,
|
||||
|
|
|
|||
|
|
@ -13,14 +13,16 @@ export default defineConfig({
|
|||
forbidOnly: !!process.env.CI,
|
||||
/* Retry on CI only */
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
/* Opt out of parallel tests on CI. */
|
||||
workers: process.env.CI ? 1 : undefined,
|
||||
/* One worker everywhere: several specs mutate global gateway state
|
||||
(router_settings, public model groups), so parallel workers race each
|
||||
other against the single shared gateway. */
|
||||
workers: 1,
|
||||
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
||||
reporter: "html",
|
||||
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
||||
use: {
|
||||
/* Base URL to use in actions like `await page.goto('/')`. */
|
||||
baseURL: "http://localhost:4000",
|
||||
baseURL: process.env.LITELLM_PROXY_URL ?? "http://localhost:4000",
|
||||
|
||||
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
|
||||
trace: "on-first-retry",
|
||||
|
|
|
|||
|
|
@ -2,15 +2,22 @@
|
|||
set -euo pipefail
|
||||
|
||||
# ================================================================
|
||||
# UI E2E Test Runner (Consolidated)
|
||||
# Starts postgres, seeds DB, starts mock + proxy, runs Playwright.
|
||||
# All tests target the proxy on port 4000 (which serves both API
|
||||
# and UI from the built Next.js static export).
|
||||
# UI E2E Test Runner
|
||||
#
|
||||
# Two modes:
|
||||
# 1. LITELLM_PROXY_URL set: run Playwright against that live gateway
|
||||
# (the same one the tests/e2e python suites target). globalSetup
|
||||
# seeds all e2e-* users/teams/keys/models through the management
|
||||
# API with LITELLM_MASTER_KEY, so the gateway only needs a master
|
||||
# key and a database (store_model_in_db: true).
|
||||
# 2. LITELLM_PROXY_URL unset: provision a disposable gateway first
|
||||
# (dockerized postgres, dashboard built from source, proxy on
|
||||
# port 4000), then run the suite against it.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_e2e.sh # Run once
|
||||
# ./run_e2e.sh --repeat-each=5 # Run each test 5 times
|
||||
# ./run_e2e.sh --headed # Run with browser visible
|
||||
# ./run_e2e.sh # provision + run
|
||||
# LITELLM_PROXY_URL=http://localhost:4000 ./run_e2e.sh # reuse a gateway
|
||||
# ./run_e2e.sh --repeat-each=5 # extra args go to playwright
|
||||
#
|
||||
# In CI (CI=true), expects:
|
||||
# - PostgreSQL already running on 127.0.0.1:5432
|
||||
|
|
@ -24,7 +31,6 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
|||
DASHBOARD_DIR="$REPO_ROOT/ui/litellm-dashboard"
|
||||
IS_CI="${CI:-false}"
|
||||
CONTAINER_NAME="litellm-e2e-postgres-$$"
|
||||
MOCK_PID=""
|
||||
PROXY_PID=""
|
||||
PROXY_LOG=""
|
||||
|
||||
|
|
@ -36,10 +42,39 @@ if [ "$IS_CI" = "false" ]; then
|
|||
[ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh"
|
||||
fi
|
||||
|
||||
# --- Cleanup on exit ---
|
||||
# Provider keys for the real-model deployments globalSetup registers via
|
||||
# /model/new; the shared harness .env is the canonical place they live.
|
||||
if [ -f "$REPO_ROOT/tests/e2e/.env" ]; then
|
||||
set -a
|
||||
source "$REPO_ROOT/tests/e2e/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
run_playwright() {
|
||||
echo "=== Installing Playwright dependencies ==="
|
||||
cd "$SCRIPT_DIR"
|
||||
npm install --silent 2>/dev/null || true
|
||||
npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium
|
||||
|
||||
echo "=== Running Playwright tests ==="
|
||||
npx playwright test --config playwright.config.ts "$@"
|
||||
}
|
||||
|
||||
# --- Mode 1: reuse an already-running gateway ---
|
||||
if [ -n "${LITELLM_PROXY_URL:-}" ]; then
|
||||
echo "=== Using existing gateway at $LITELLM_PROXY_URL ==="
|
||||
export LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY:-sk-1234}"
|
||||
curl -fs "$LITELLM_PROXY_URL/health/liveliness" >/dev/null || {
|
||||
echo "Error: no live proxy at $LITELLM_PROXY_URL"
|
||||
exit 1
|
||||
}
|
||||
run_playwright "$@"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# --- Mode 2: provision a disposable gateway ---
|
||||
cleanup() {
|
||||
echo "Cleaning up..."
|
||||
[ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true
|
||||
[ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true
|
||||
[ -n "$PROXY_LOG" ] && rm -f "$PROXY_LOG" || true
|
||||
if [ "$IS_CI" = "false" ]; then
|
||||
|
|
@ -56,10 +91,10 @@ done
|
|||
|
||||
# --- Database setup ---
|
||||
if [ "$IS_CI" = "false" ]; then
|
||||
for cmd in docker psql; do
|
||||
for cmd in docker; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
|
||||
done
|
||||
for port in 4000 5432 8090; do
|
||||
for port in 4000 5432; do
|
||||
if lsof -ti ":$port" >/dev/null 2>&1; then
|
||||
echo "Error: port $port is in use"
|
||||
exit 1
|
||||
|
|
@ -79,7 +114,7 @@ if [ "$IS_CI" = "false" ]; then
|
|||
|
||||
echo "Waiting for PostgreSQL..."
|
||||
for i in $(seq 1 30); do
|
||||
if PGPASSWORD="$POSTGRES_PASSWORD" pg_isready -h 127.0.0.1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then
|
||||
if docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
|
|
@ -91,7 +126,6 @@ fi
|
|||
|
||||
# --- Credentials ---
|
||||
export LITELLM_MASTER_KEY="sk-1234"
|
||||
export MOCK_LLM_URL="http://127.0.0.1:8090/v1"
|
||||
export DISABLE_SCHEMA_UPDATE="true"
|
||||
# Ensure the proxy serves UI at /ui (not behind a subpath)
|
||||
export SERVER_ROOT_PATH=""
|
||||
|
|
@ -133,16 +167,6 @@ uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
|
|||
echo "=== Pushing Prisma schema to database ==="
|
||||
uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
|
||||
|
||||
# --- Mock LLM server ---
|
||||
echo "=== Starting mock LLM server ==="
|
||||
uv run --no-sync python "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" &
|
||||
MOCK_PID=$!
|
||||
|
||||
for i in $(seq 1 15); do
|
||||
if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# --- LiteLLM proxy ---
|
||||
echo "=== Starting LiteLLM proxy ==="
|
||||
cd "$REPO_ROOT"
|
||||
|
|
@ -174,25 +198,5 @@ if [ "$PROXY_READY" -ne 1 ]; then
|
|||
fi
|
||||
echo "Proxy is ready."
|
||||
|
||||
# --- Seed database ---
|
||||
echo "=== Seeding database ==="
|
||||
DB_USER=$(echo "$DATABASE_URL" | sed -n 's|.*://\([^:]*\):.*|\1|p')
|
||||
DB_PASS=$(echo "$DATABASE_URL" | sed -n 's|.*://[^:]*:\([^@]*\)@.*|\1|p')
|
||||
DB_HOST=$(echo "$DATABASE_URL" | sed -n 's|.*@\([^:]*\):.*|\1|p')
|
||||
DB_PORT=$(echo "$DATABASE_URL" | sed -n 's|.*:\([0-9]*\)/.*|\1|p')
|
||||
DB_NAME=$(echo "$DATABASE_URL" | sed -n 's|.*/\([^?]*\).*|\1|p')
|
||||
|
||||
PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
|
||||
-f "$SCRIPT_DIR/fixtures/seed.sql"
|
||||
|
||||
# --- Playwright ---
|
||||
echo "=== Installing Playwright dependencies ==="
|
||||
cd "$SCRIPT_DIR"
|
||||
npm install --silent 2>/dev/null || true
|
||||
npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium
|
||||
|
||||
echo "=== Running Playwright tests ==="
|
||||
npx playwright test --config playwright.config.ts "$@"
|
||||
EXIT_CODE=$?
|
||||
|
||||
exit $EXIT_CODE
|
||||
run_playwright "$@"
|
||||
exit $?
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import { test, expect } from "@playwright/test";
|
||||
import { PROXY_BASE_URL } from "../../constants";
|
||||
|
||||
test.describe("Authentication Checks", () => {
|
||||
test("should redirect unauthenticated user from a protected page", async ({ page }) => {
|
||||
const protectedPageUrl = "http://localhost:4000/ui?page=llm-playground";
|
||||
const protectedPageUrl = `${PROXY_BASE_URL}/ui?page=llm-playground`;
|
||||
await page.goto(protectedPageUrl, { waitUntil: "domcontentloaded" });
|
||||
await expect(page).toHaveURL(/\/ui\/login/);
|
||||
await expect(page.getByRole("heading", { name: "Login" })).toBeVisible();
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { expect, test } from "@playwright/test";
|
||||
import { users } from "../../fixtures/users";
|
||||
import { Role } from "../../fixtures/roles";
|
||||
import { PROXY_BASE_URL } from "../../constants";
|
||||
|
||||
test("user can log in", async ({ page }) => {
|
||||
await page.goto("http://localhost:4000/ui/login");
|
||||
await page.goto(`${PROXY_BASE_URL}/ui/login`);
|
||||
await page.getByPlaceholder("Enter your username").fill(users[Role.ProxyAdmin].email);
|
||||
await page.getByPlaceholder("Enter your password").fill(users[Role.ProxyAdmin].password);
|
||||
const loginButton = page.getByRole("button", { name: "Login", exact: true });
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { expect, test } from "@playwright/test";
|
||||
import { PROXY_BASE_URL } from "../../constants";
|
||||
|
||||
// Driven by the SERVER_ROOT_PATH env var injected by the workflow; the container
|
||||
// is booted with the same value, so the asset paths and the runtime config it
|
||||
|
|
@ -24,7 +25,7 @@ test("unauth redirect preserves SERVER_ROOT_PATH prefix", async ({ page }) => {
|
|||
|
||||
await page.context().clearCookies();
|
||||
|
||||
await page.goto(`http://localhost:4000${ROOT_PATH}/ui/?page=virtual-keys`);
|
||||
await page.goto(`${PROXY_BASE_URL}${ROOT_PATH}/ui/?page=virtual-keys`);
|
||||
|
||||
await page.waitForURL((url) => url.pathname.includes("/ui/login"), { timeout: 15_000 });
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { test, expect } from "@playwright/test";
|
||||
import { test, expect, APIRequestContext } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { navigateToPage } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
|
|
@ -8,9 +8,34 @@ import { Page } from "../../fixtures/pages";
|
|||
// — SSE / stdio / OpenAPI transports, API Key / Bearer / OAuth2 / Basic / Token
|
||||
// / AWS SigV4 auth, edit/delete, BYOK credentials, tool list/call (needs a real
|
||||
// or mocked MCP server in the e2e fixture stack), and access-group permissions.
|
||||
|
||||
// Leftover e2e_mcp_* servers point at a dead URL, and the MCP page's health
|
||||
// checks against them keep the network busy long enough that navigation's
|
||||
// networkidle wait times out on the next run against a persistent gateway.
|
||||
async function purgeE2eMcpServers(request: APIRequestContext): Promise<void> {
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234";
|
||||
const auth = { Authorization: `Bearer ${masterKey}` };
|
||||
const res = await request.get("/v1/mcp/server", { headers: auth });
|
||||
if (!res.ok()) return;
|
||||
const servers: Array<{ server_id: string; server_name?: string }> = await res.json();
|
||||
for (const server of servers) {
|
||||
if (server.server_name?.startsWith("e2e_mcp_")) {
|
||||
await request.delete(`/v1/mcp/server/${server.server_id}`, { headers: auth });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test.describe("MCP Servers", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test.beforeEach(async ({ request }) => {
|
||||
await purgeE2eMcpServers(request);
|
||||
});
|
||||
|
||||
test.afterEach(async ({ request }) => {
|
||||
await purgeE2eMcpServers(request);
|
||||
});
|
||||
|
||||
test("Add a custom MCP server via the discovery → custom form", async ({ page }) => {
|
||||
await navigateToPage(page, Page.McpServers);
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,18 @@ import { Page } from "../../fixtures/pages";
|
|||
test.describe("AI Hub (internal admin view)", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
// Reset public model groups so the run is idempotent against a persistent
|
||||
// gateway: make_public replaces the whole list, and once a prior run made
|
||||
// every group public the modal's Select All has zero rows left to pick.
|
||||
test.beforeEach(async ({ page }) => {
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234";
|
||||
const res = await page.request.post("/model_group/make_public", {
|
||||
headers: { Authorization: `Bearer ${masterKey}` },
|
||||
data: { model_groups: [] },
|
||||
});
|
||||
expect(res.ok(), `resetting public model groups failed: ${res.status()}`).toBe(true);
|
||||
});
|
||||
|
||||
test("Make models public via the multi-step modal", async ({ page }) => {
|
||||
await navigateToPage(page, Page.ModelHubTable);
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
E2E_UPDATE_LIMITS_KEY_ALIAS,
|
||||
E2E_INTERNAL_USER_KEY_ALIAS,
|
||||
E2E_TEAM_CRUD_ALIAS,
|
||||
E2E_UI_OPENAI_MODEL,
|
||||
} from "../../constants";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
|
||||
|
|
@ -166,7 +167,7 @@ test.describe("Proxy Admin - Keys", () => {
|
|||
// Open the model multi-select and pick a single specific model. Use
|
||||
// getByRole("option", ...) to avoid the strict-mode collision between
|
||||
// the option container and its inner text node.
|
||||
const modelName = "fake-openai-gpt-4";
|
||||
const modelName = E2E_UI_OPENAI_MODEL;
|
||||
await page.locator(".ant-select-selection-overflow").click();
|
||||
const option = page.locator(".ant-select-dropdown:visible").getByRole("option", { name: modelName, exact: true });
|
||||
await option.waitFor({ state: "attached" });
|
||||
|
|
@ -182,8 +183,8 @@ test.describe("Proxy Admin - Keys", () => {
|
|||
|
||||
// Grab the new key from the success modal (rendered inside a <pre>) and
|
||||
// verify it can call /chat/completions for the model it was scoped to.
|
||||
// The mock LLM server (fixtures/mock_llm_server/server.py) replies with
|
||||
// a fixed "This is a mock response." body.
|
||||
// The deployment routes to a real provider, so this costs a fraction of
|
||||
// a cent and proves the whole key -> router -> provider path.
|
||||
const apiKey = (await page.locator(".ant-modal:visible pre").innerText()).trim();
|
||||
expect(apiKey).toMatch(/^sk-/);
|
||||
|
||||
|
|
@ -191,12 +192,12 @@ test.describe("Proxy Admin - Keys", () => {
|
|||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
data: {
|
||||
model: modelName,
|
||||
messages: [{ role: "user", content: "ping" }],
|
||||
messages: [{ role: "user", content: "Reply with the single word: pong" }],
|
||||
},
|
||||
});
|
||||
expect(response.status()).toBe(200);
|
||||
const body = await response.json();
|
||||
expect(body.choices?.[0]?.message?.content).toBe("This is a mock response.");
|
||||
expect(body.choices?.[0]?.message?.content?.length).toBeGreaterThan(0);
|
||||
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ import {
|
|||
E2E_TEAM_DELETE_ALIAS,
|
||||
E2E_TEAM_NO_ADMIN_ID,
|
||||
E2E_TEAM_ORG_ID,
|
||||
E2E_UI_ANTHROPIC_MODEL,
|
||||
E2E_UI_OPENAI_MODEL,
|
||||
PROXY_BASE_URL,
|
||||
} from "../../constants";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
|
||||
|
|
@ -127,12 +130,12 @@ test.describe("Proxy Admin - Teams", () => {
|
|||
|
||||
test("Edit team model selection", async ({ page, request }) => {
|
||||
// Restore the seeded models via API in case a prior run (or a CI retry)
|
||||
// left this team mutated — the assertion below requires fake-anthropic-claude
|
||||
// to be present.
|
||||
// left this team mutated — the assertion below requires the anthropic
|
||||
// deployment to be present.
|
||||
const masterKey = process.env.LITELLM_MASTER_KEY || "sk-1234";
|
||||
const seededModels = ["fake-openai-gpt-4", "fake-anthropic-claude"];
|
||||
const seededModels = [E2E_UI_OPENAI_MODEL, E2E_UI_ANTHROPIC_MODEL];
|
||||
const restore = async () => {
|
||||
const res = await request.post("http://localhost:4000/team/update", {
|
||||
const res = await request.post(`${PROXY_BASE_URL}/team/update`, {
|
||||
headers: { Authorization: `Bearer ${masterKey}` },
|
||||
data: { team_id: E2E_TEAM_CRUD_ID, models: seededModels },
|
||||
});
|
||||
|
|
@ -156,7 +159,7 @@ test.describe("Proxy Admin - Teams", () => {
|
|||
|
||||
const anthropicTag = modelsSelect
|
||||
.locator(".ant-select-selection-item")
|
||||
.filter({ hasText: "fake-anthropic-claude" });
|
||||
.filter({ hasText: E2E_UI_ANTHROPIC_MODEL });
|
||||
await expect(anthropicTag).toBeVisible({ timeout: 5_000 });
|
||||
await anthropicTag.locator(".ant-select-selection-item-remove").click();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { test, expect } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { ADMIN_STORAGE_PATH, E2E_UI_ANTHROPIC_MODEL, E2E_UI_OPENAI_MODEL, PROXY_BASE_URL } from "../../constants";
|
||||
import { navigateToPage } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { Role, users } from "../../fixtures/users";
|
||||
|
|
@ -12,8 +12,8 @@ import type { components } from "../../../../../ui/litellm-dashboard/src/lib/htt
|
|||
// echoes the whole settings object, so they must not run concurrently.
|
||||
test.describe.configure({ mode: "serial" });
|
||||
|
||||
const PRIMARY = "fake-openai-gpt-4";
|
||||
const FALLBACK = "fake-anthropic-claude";
|
||||
const PRIMARY = E2E_UI_OPENAI_MODEL;
|
||||
const FALLBACK = E2E_UI_ANTHROPIC_MODEL;
|
||||
|
||||
/**
|
||||
* Wipe any fallbacks for the primary model so the test is idempotent across
|
||||
|
|
@ -23,7 +23,7 @@ async function clearFallbackForPrimary(request: import("@playwright/test").APIRe
|
|||
const masterKey = users[Role.ProxyAdmin].password;
|
||||
const auth = { Authorization: `Bearer ${masterKey}` };
|
||||
|
||||
const current = await request.get("http://localhost:4000/get/config/callbacks", { headers: auth });
|
||||
const current = await request.get(`${PROXY_BASE_URL}/get/config/callbacks`, { headers: auth });
|
||||
if (!current.ok()) return;
|
||||
const body = await current.json();
|
||||
const router = body?.router_settings ?? {};
|
||||
|
|
@ -31,7 +31,7 @@ async function clearFallbackForPrimary(request: import("@playwright/test").APIRe
|
|||
const next = existing.filter((entry) => !(entry && PRIMARY in entry));
|
||||
if (next.length === existing.length) return;
|
||||
|
||||
await request.post("http://localhost:4000/config/update", {
|
||||
await request.post(`${PROXY_BASE_URL}/config/update`, {
|
||||
headers: auth,
|
||||
data: { router_settings: { ...router, fallbacks: next } },
|
||||
});
|
||||
|
|
@ -111,7 +111,7 @@ test.describe("Router Settings - Fallbacks", () => {
|
|||
type ConfigYAML = components["schemas"]["ConfigYAML"];
|
||||
type RouterSettingsResponse = components["schemas"]["RouterSettingsResponse"];
|
||||
|
||||
const BASE_URL = "http://localhost:4000";
|
||||
const BASE_URL = PROXY_BASE_URL;
|
||||
const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` };
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue