mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
274a489c46
The guardrail section of the release checklist is done by hand every cut: create a Presidio guardrail through the wizard, send a sentence with PII from the playground, then open Logs and check the guardrail caught it. Nothing covered that path, so a break anywhere along it surfaced only when someone happened to repeat the steps. Adds a spec that walks it once and turns the eyeball checks into assertions. The strongest of them is the leak check: it reads the request back and fails if the stored prompt still carries the raw address or number, which is what the manual step is really looking for. The stack gains a Presidio stand-in that answers the two routes the guardrail calls, detecting a fixed regex set with a Luhn check on card numbers. Real Presidio's detection quality is Presidio's business, and pinning the UI lane to it would mean two heavy containers with spaCy models on every CI run for a test that is about LiteLLM's integration. The real analyzer stays covered in the Python lane. The stand-in returns the same entities at the same spans as the real one for the checklist's sentence, and driving the real guardrail against it produces the same record shape, so a test written against it is written against the product's real behavior. Making the stand-in return the text unmasked turns the spec red on the raw address reaching the spend log, so the leak assertion reads live data. run_e2e.sh and both CircleCI UI jobs start the stand-in alongside the mock LLM. Its port is overridable like the others so two checkouts can run at once.
54 lines
2.7 KiB
TypeScript
54 lines
2.7 KiB
TypeScript
import * as path from "path";
|
|
|
|
export const UI_BASE_URL = (
|
|
process.env.E2E_UI_BASE_URL ||
|
|
process.env.LITELLM_PROXY_URL ||
|
|
"http://localhost:4000"
|
|
).replace(/\/+$/, "");
|
|
|
|
// Directory every artifact this suite writes must land in: storage states,
|
|
// failure screenshots, playwright output. A local `./run_e2e.sh` run has a
|
|
// writable cwd, so it keeps the historical behavior of writing beside the
|
|
// suite. In the packaged e2e image the suite ships on a read-only filesystem
|
|
// (/app/e2e/ui), so bare relative paths raise EROFS/ENOENT and the run dies in
|
|
// globalSetup before a single test executes. Point E2E_UI_ARTIFACT_DIR at a
|
|
// writable path (the image runner already exports TMPDIR) to relocate them.
|
|
export const ARTIFACT_DIR = process.env.E2E_UI_ARTIFACT_DIR || ".";
|
|
|
|
export const MOCK_PRESIDIO_URL = (process.env.E2E_MOCK_PRESIDIO_URL || "http://127.0.0.1:8091").replace(/\/+$/, "");
|
|
|
|
const storagePath = (name: string): string => path.join(ARTIFACT_DIR, name);
|
|
|
|
// Storage state paths for each role
|
|
export const ADMIN_STORAGE_PATH = storagePath("admin.storageState.json");
|
|
export const ADMIN_VIEWER_STORAGE_PATH = storagePath("adminViewer.storageState.json");
|
|
export const INTERNAL_USER_STORAGE_PATH = storagePath("internalUser.storageState.json");
|
|
export const INTERNAL_VIEWER_STORAGE_PATH = storagePath("internalViewer.storageState.json");
|
|
export const TEAM_ADMIN_STORAGE_PATH = storagePath("teamAdmin.storageState.json");
|
|
|
|
// Seeded user identities (match seed.sql)
|
|
export const E2E_PROXY_ADMIN_USER_ID = "e2e-proxy-admin";
|
|
export const E2E_PROXY_ADMIN_EMAIL = "admin@test.local";
|
|
export const E2E_INTERNAL_USER_ID = "e2e-internal-user";
|
|
export const E2E_INTERNAL_USER_EMAIL = "internal@test.local";
|
|
export const E2E_TEAM_ADMIN_USER_ID = "e2e-team-admin";
|
|
export const E2E_SEEDED_USER_PASSWORD = "E2e-Test-Pass-2026!";
|
|
|
|
// Key aliases for seeded test keys (match seed.sql)
|
|
export const E2E_UPDATE_LIMITS_KEY_ALIAS = "e2eUpdateLimitsKey";
|
|
export const E2E_DELETE_KEY_ALIAS = "e2eDeleteKey";
|
|
export const E2E_REGENERATE_KEY_ALIAS = "e2eRegenerateKey";
|
|
export const E2E_INTERNAL_USER_KEY_ALIAS = "e2eInternalUserKey";
|
|
export const E2E_VIEWER_KEY_ALIAS = "e2eViewerKey";
|
|
|
|
// Team identifiers (match seed.sql)
|
|
export const E2E_TEAM_CRUD_ID = "e2e-team-crud";
|
|
export const E2E_TEAM_CRUD_ALIAS = "E2E Team CRUD";
|
|
export const E2E_TEAM_DELETE_ID = "e2e-team-delete";
|
|
export const E2E_TEAM_DELETE_ALIAS = "E2E Team Delete";
|
|
export const E2E_TEAM_ORG_ID = "e2e-team-org";
|
|
export const E2E_TEAM_ORG_ALIAS = "E2E Team In Org";
|
|
export const E2E_TEAM_NO_ADMIN_ID = "e2e-team-no-admin";
|
|
export const E2E_TEAM_NO_ADMIN_ALIAS = "E2E Team No Admin";
|
|
export const E2E_TEAM_KEYGEN_ID = "e2e-team-keygen";
|
|
export const E2E_TEAM_KEYGEN_ALIAS = "E2E Team Keygen";
|