From 274a489c463452c78e3e69db2017ec2e3ad59315 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 6 Sep 2026 02:36:46 -0700 Subject: [PATCH 1/5] test(e2e/ui): automate the RC checklist's Presidio guardrail walk The guardrail section of the release checklist is done by hand every cut: create a Presidio guardrail through the wizard, send a sentence with PII from the playground, then open Logs and check the guardrail caught it. Nothing covered that path, so a break anywhere along it surfaced only when someone happened to repeat the steps. Adds a spec that walks it once and turns the eyeball checks into assertions. The strongest of them is the leak check: it reads the request back and fails if the stored prompt still carries the raw address or number, which is what the manual step is really looking for. The stack gains a Presidio stand-in that answers the two routes the guardrail calls, detecting a fixed regex set with a Luhn check on card numbers. Real Presidio's detection quality is Presidio's business, and pinning the UI lane to it would mean two heavy containers with spaCy models on every CI run for a test that is about LiteLLM's integration. The real analyzer stays covered in the Python lane. The stand-in returns the same entities at the same spans as the real one for the checklist's sentence, and driving the real guardrail against it produces the same record shape, so a test written against it is written against the product's real behavior. Making the stand-in return the text unmasked turns the spec red on the raw address reaching the spend log, so the leak assertion reads live data. run_e2e.sh and both CircleCI UI jobs start the stand-in alongside the mock LLM. Its port is overridable like the others so two checkouts can run at once. --- .circleci/config.yml | 8 + tests/e2e/ui/constants.ts | 2 + .../fixtures/mock_presidio_server/server.py | 107 ++++++++++++ tests/e2e/ui/run_e2e.sh | 21 ++- .../guardrails/presidioUserStory.spec.ts | 158 ++++++++++++++++++ 5 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/ui/fixtures/mock_presidio_server/server.py create mode 100644 tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e368a3debe..276555c0cf7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2648,6 +2648,10 @@ jobs: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py background: true + - run: + name: Start mock Presidio server + command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py + background: true - run: name: Start LiteLLM proxy environment: @@ -2778,6 +2782,10 @@ jobs: name: Start mock LLM server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py background: true + - run: + name: Start mock Presidio server + command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py + background: true - run: name: Start LiteLLM proxy under a server root path environment: diff --git a/tests/e2e/ui/constants.ts b/tests/e2e/ui/constants.ts index 3a62252915c..71774c95d24 100644 --- a/tests/e2e/ui/constants.ts +++ b/tests/e2e/ui/constants.ts @@ -15,6 +15,8 @@ export const UI_BASE_URL = ( // writable path (the image runner already exports TMPDIR) to relocate them. export const ARTIFACT_DIR = process.env.E2E_UI_ARTIFACT_DIR || "."; +export const MOCK_PRESIDIO_URL = (process.env.E2E_MOCK_PRESIDIO_URL || "http://127.0.0.1:8091").replace(/\/+$/, ""); + const storagePath = (name: string): string => path.join(ARTIFACT_DIR, name); // Storage state paths for each role diff --git a/tests/e2e/ui/fixtures/mock_presidio_server/server.py b/tests/e2e/ui/fixtures/mock_presidio_server/server.py new file mode 100644 index 00000000000..0fc97c4a065 --- /dev/null +++ b/tests/e2e/ui/fixtures/mock_presidio_server/server.py @@ -0,0 +1,107 @@ +""" +Mock Presidio analyzer + anonymizer for UI e2e tests. +Serves POST /analyze and POST /anonymize over a fixed set of regex recognizers. +""" + +import os +import re + +import uvicorn +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware + + +def _luhn_ok(candidate): + digits = [int(c) for c in candidate if c.isdigit()] + doubled = [d * 2 - 9 if d * 2 > 9 else d * 2 for d in digits[-2::-2]] + return len(digits) >= 13 and (sum(digits[::-1][::2]) + sum(doubled)) % 10 == 0 + + +RECOGNIZERS = ( + ("EMAIL_ADDRESS", re.compile(r"\b[\w.+-]+@[\w-]+\.[\w.-]+\b"), 1.0, None), + ("CREDIT_CARD", re.compile(r"\b(?:\d[ -]?){13,19}\b"), 1.0, _luhn_ok), + ("US_SSN", re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), 0.85, None), + ("PHONE_NUMBER", re.compile(r"\b(?:\+?\d{1,2}[ .-]?)?(?:\(\d{3}\)|\d{3})[ .-]?\d{3}[ .-]?\d{4}\b"), 0.75, None), + ("IP_ADDRESS", re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), 0.6, None), + ("URL", re.compile(r"\bhttps?://[^\s]+"), 0.5, None), +) + +app = FastAPI(title="Mock Presidio Server") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +def _detect(text, wanted): + found = [ + {"entity_type": name, "start": m.start(), "end": m.end(), "score": score} + for name, pattern, score, validate in RECOGNIZERS + if wanted is None or name in wanted + for m in pattern.finditer(text) + if validate is None or validate(m.group()) + ] + ranked = sorted(found, key=lambda r: (-r["score"], r["start"])) + kept = [] + for candidate in ranked: + overlaps = any(candidate["start"] < k["end"] and k["start"] < candidate["end"] for k in kept) + if not overlaps: + kept.append(candidate) + return sorted(kept, key=lambda r: r["start"]) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + +@app.post("/analyze") +async def analyze(request: Request): + body = await request.json() + text = body.get("text", "") or "" + entities = body.get("entities") + wanted = set(entities) if entities else None + threshold = body.get("score_threshold") or 0.0 + return [ + {**result, "analysis_explanation": None, "recognition_metadata": {"recognizer_name": "MockRecognizer"}} + for result in _detect(text, wanted) + if result["score"] >= threshold + ] + + +@app.post("/anonymize") +async def anonymize(request: Request): + body = await request.json() + text = body.get("text", "") or "" + results = sorted(body.get("analyzer_results") or [], key=lambda r: r["start"]) + + pieces = [] + items = [] + cursor = 0 + for result in results: + start, end = result["start"], result["end"] + if start < cursor: + continue + placeholder = f"<{result['entity_type']}>" + pieces.append(text[cursor:start]) + masked_start = sum(len(p) for p in pieces) + pieces.append(placeholder) + items.append( + { + "operator": "replace", + "entity_type": result["entity_type"], + "start": masked_start, + "end": masked_start + len(placeholder), + "text": placeholder, + } + ) + cursor = end + pieces.append(text[cursor:]) + + return {"text": "".join(pieces), "items": list(reversed(items))} + + +if __name__ == "__main__": + uvicorn.run(app, host="127.0.0.1", port=int(os.environ.get("MOCK_PRESIDIO_PORT", "8091"))) diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index a369d8c8f7a..ad0454b6347 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -14,7 +14,7 @@ set -euo pipefail # # Ports default to 4000 / 5432 / 8090 and can be moved when another checkout # already holds them: -# PROXY_PORT=4100 POSTGRES_PORT=5532 MOCK_LLM_PORT=8190 ./run_e2e.sh +# PROXY_PORT=4100 POSTGRES_PORT=5532 MOCK_LLM_PORT=8190 MOCK_PRESIDIO_PORT=8191 ./run_e2e.sh # # In CI (CI=true), expects: # - PostgreSQL already running on 127.0.0.1:5432 @@ -29,6 +29,7 @@ DASHBOARD_DIR="$REPO_ROOT/ui/litellm-dashboard" IS_CI="${CI:-false}" CONTAINER_NAME="litellm-e2e-postgres-$$" MOCK_PID="" +MOCK_PRESIDIO_PID="" PROXY_PID="" PROXY_LOG="" @@ -40,7 +41,8 @@ PROXY_LOG="" PROXY_PORT="${PROXY_PORT:-4000}" POSTGRES_PORT="${POSTGRES_PORT:-5432}" MOCK_LLM_PORT="${MOCK_LLM_PORT:-8090}" -export MOCK_LLM_PORT +MOCK_PRESIDIO_PORT="${MOCK_PRESIDIO_PORT:-8091}" +export MOCK_LLM_PORT MOCK_PRESIDIO_PORT # --- Ensure common tool paths are available (local dev only) --- if [ "$IS_CI" = "false" ]; then @@ -82,6 +84,7 @@ fi cleanup() { echo "Cleaning up..." [ -n "$MOCK_PID" ] && kill "$MOCK_PID" 2>/dev/null || true + [ -n "$MOCK_PRESIDIO_PID" ] && kill "$MOCK_PRESIDIO_PID" 2>/dev/null || true [ -n "$PROXY_PID" ] && kill "$PROXY_PID" 2>/dev/null || true [ -n "$PROXY_LOG" ] && rm -f "$PROXY_LOG" || true if [ "$IS_CI" = "false" ]; then @@ -110,9 +113,9 @@ if [ "$IS_CI" = "false" ]; then # to someone else's :5432 (a psql session, a running app, a Prisma engine # talking to a remote database) aborts the run with "port 5432 is in use" # while nothing is actually bound locally. - for port in "$PROXY_PORT" "$POSTGRES_PORT" "$MOCK_LLM_PORT"; do + for port in "$PROXY_PORT" "$POSTGRES_PORT" "$MOCK_LLM_PORT" "$MOCK_PRESIDIO_PORT"; do if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then - echo "Error: port $port is in use (override with PROXY_PORT / POSTGRES_PORT / MOCK_LLM_PORT)" + echo "Error: port $port is in use (override with PROXY_PORT / POSTGRES_PORT / MOCK_LLM_PORT / MOCK_PRESIDIO_PORT)" exit 1 fi done @@ -143,6 +146,7 @@ fi # --- Credentials --- export LITELLM_MASTER_KEY="sk-1234" export MOCK_LLM_URL="http://127.0.0.1:${MOCK_LLM_PORT}/v1" +export E2E_MOCK_PRESIDIO_URL="http://127.0.0.1:${MOCK_PRESIDIO_PORT}" export DISABLE_SCHEMA_UPDATE="true" # The suite resolves its target from E2E_UI_BASE_URL (constants.ts), which # otherwise defaults to :4000 -- so without this a relocated stack would be @@ -204,6 +208,15 @@ for i in $(seq 1 15); do sleep 1 done +echo "=== Starting mock Presidio server ===" +uv run --no-sync python "$SCRIPT_DIR/fixtures/mock_presidio_server/server.py" & +MOCK_PRESIDIO_PID=$! + +for i in $(seq 1 15); do + if curl -sf http://127.0.0.1:${MOCK_PRESIDIO_PORT}/health >/dev/null 2>&1; then break; fi + sleep 1 +done + # --- LiteLLM proxy --- echo "=== Starting LiteLLM proxy ===" cd "$REPO_ROOT" diff --git a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts new file mode 100644 index 00000000000..975b343287a --- /dev/null +++ b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts @@ -0,0 +1,158 @@ +import { test, expect, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH, MOCK_PRESIDIO_URL } from "../../constants"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { CHAT_MODEL_A, masterKey, rootPath, waitForSpendLogByPrompt } from "../../helpers/traffic"; +import { openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground"; + +const RAW_EMAIL = "jane.doe@example.com"; +const RAW_PHONE = "555-867-5309"; + +const visibleTestId = (page: PlaywrightPage, id: string) => page.getByTestId(id).filter({ visible: true }); + +const requestLogsRows = (page: PlaywrightPage) => + page.locator("table").filter({ visible: true }).first().locator("tbody tr"); + +async function createPresidioGuardrail(page: PlaywrightPage, guardrailName: string): Promise { + await navigateToPage(page, Page.Guardrails); + await dismissFeedbackPopup(page); + + await page.getByRole("button", { name: /Add New Guardrail/i }).click(); + await page.getByRole("menuitem", { name: "Add Provider Guardrail" }).click(); + + const dialog = page.getByRole("dialog", { name: "Create guardrail" }); + await expect(dialog).toBeVisible({ timeout: 10_000 }); + + await dialog.getByLabel("Guardrail Name").fill(guardrailName); + + const providerSelect = dialog.getByRole("combobox", { name: "Guardrail Provider" }); + await providerSelect.click(); + await providerSelect.fill("Presidio"); + await page.getByRole("option", { name: "Presidio PII" }).click(); + + await dialog.getByLabel("Mode", { exact: true }).click(); + await page.keyboard.type("pre_call"); + await expect(page.getByRole("option", { name: "pre_call" })).toBeAttached({ timeout: 5_000 }); + await page.keyboard.press("Enter"); + await expect(dialog.getByText("pre_call", { exact: true })).toBeVisible({ timeout: 5_000 }); + await dialog.getByText("Create guardrail", { exact: true }).click(); + + await dialog.getByLabel("presidio_analyzer_api_base").fill(MOCK_PRESIDIO_URL); + await dialog.getByLabel("presidio_anonymizer_api_base").fill(MOCK_PRESIDIO_URL); + + await dialog.getByRole("button", { name: "Next" }).click(); + await expect(dialog.getByText("Configure PII Protection")).toBeVisible({ timeout: 10_000 }); + await dialog.getByRole("button", { name: "Select All & Mask" }).click(); + + await dialog.getByRole("button", { name: "Create Guardrail" }).click(); + await expect(page.getByText("Guardrail created successfully").first()).toBeVisible({ timeout: 15_000 }); + await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(1, { timeout: 15_000 }); +} + +async function deleteGuardrail(page: PlaywrightPage, guardrailName: string): Promise { + await navigateToPage(page, Page.Guardrails); + await dismissFeedbackPopup(page); + const row = page.getByRole("row").filter({ hasText: guardrailName }); + await expect(row).toHaveCount(1, { timeout: 15_000 }); + await row.getByRole("button", { name: "Open guardrail actions" }).click(); + await page.getByRole("menuitem", { name: "Delete" }).click(); + const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" }); + await expect(deleteModal).toBeVisible({ timeout: 5_000 }); + await deleteModal.getByRole("button", { name: "Delete", exact: true }).click(); + await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 }); +} + +test.describe("Presidio PII guardrail, end to end from the dashboard", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("masks PII sent from the Playground and shows the run in Logs", async ({ page, request }) => { + const guardrailName = `e2e-presidio-story-${Date.now()}`; + const marker = `case-ref-${Math.random().toString(36).slice(2, 10)}`; + const prompt = `${marker}. Email me at ${RAW_EMAIL} or call ${RAW_PHONE}.`; + + await createPresidioGuardrail(page, guardrailName); + + await openPlayground(page); + await selectModel(page, CHAT_MODEL_A); + + const guardrailSelect = onlyVisible(page.getByPlaceholder("Select guardrails")); + await expect(guardrailSelect).toBeVisible({ timeout: 20_000 }); + await guardrailSelect.click(); + await guardrailSelect.fill(guardrailName); + await onlyVisible(page.getByRole("option", { name: guardrailName, exact: true })).click({ timeout: 20_000 }); + await page.keyboard.press("Escape"); + + const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false })); + await expect(input).toBeVisible({ timeout: 15_000 }); + + await expect + .poll( + async () => { + await input.fill(prompt); + await sendButton(page).click(); + const res = await request.get(`${rootPath()}/spend/logs`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + if (!res.ok()) return false; + const rows: { metadata?: { applied_guardrails?: string[] } }[] = await res.json(); + return (Array.isArray(rows) ? rows : []).some((row) => + (row.metadata?.applied_guardrails ?? []).includes(guardrailName), + ); + }, + { + message: `the playground never produced a request that ran ${guardrailName}`, + timeout: 90_000, + intervals: [5_000], + }, + ) + .toBe(true); + + const requestId = await waitForSpendLogByPrompt(request, marker); + + const stored = await request.get(`${rootPath()}/spend/logs?request_id=${requestId}`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + expect(stored.ok(), `spend log read failed: ${stored.status()}`).toBe(true); + const storedBody = JSON.stringify(await stored.json()); + expect(storedBody, "the raw email reached the spend log, so the prompt was stored unmasked").not.toContain( + RAW_EMAIL, + ); + expect(storedBody, "the raw phone number reached the spend log, so the prompt was stored unmasked").not.toContain( + RAW_PHONE, + ); + expect(storedBody, "the stored prompt carries no EMAIL_ADDRESS placeholder, so nothing was masked").toContain( + "", + ); + expect(storedBody, "the stored prompt carries no PHONE_NUMBER placeholder, so nothing was masked").toContain( + "", + ); + + await navigateToPage(page, Page.Logs); + await dismissFeedbackPopup(page); + const search = visibleTestId(page, "datatable-search"); + await expect(search).toBeVisible({ timeout: 20_000 }); + await search.fill(requestId); + const row = requestLogsRows(page).filter({ hasText: requestId }); + await expect(row, `no logs row for request ${requestId}`).toHaveCount(1, { timeout: 30_000 }); + await row.click(); + + const drawer = page.getByRole("dialog").first(); + await expect(drawer.getByText("Guardrails & Policy Compliance")).toBeVisible({ timeout: 20_000 }); + await expect(drawer.getByText(`Pre-call guardrail: ${guardrailName}`).first()).toBeVisible({ timeout: 20_000 }); + await expect(drawer.getByText(`${marker}. Email me at or call .`)).toBeVisible({ + timeout: 20_000, + }); + + await drawer.getByText("2 matched").first().click(); + await expect(drawer.getByText("Detected Entities (2)").first()).toBeVisible({ timeout: 10_000 }); + await expect(drawer.getByText("EMAIL_ADDRESS").first()).toBeVisible({ timeout: 10_000 }); + await expect(drawer.getByText("PHONE_NUMBER").first()).toBeVisible({ timeout: 10_000 }); + await expect(drawer.getByText("Score: 1.00").first()).toBeVisible({ timeout: 10_000 }); + await expect(drawer.getByText("Score: 0.75").first()).toBeVisible({ timeout: 10_000 }); + + await expect(drawer.getByText(RAW_EMAIL)).toHaveCount(0); + await expect(drawer.getByText(RAW_PHONE)).toHaveCount(0); + + await deleteGuardrail(page, guardrailName); + }); +}); From b0e53bfbe4e8bbeb1e3941d23728e1b5f8beaeaa Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 6 Sep 2026 03:13:09 -0700 Subject: [PATCH 2/5] fix(e2e/ui): fail the run when the Presidio fixture never comes up Both readiness loops broke out on success and fell through on timeout, so a mock Presidio server that failed to bind left the run going with nothing serving /analyze. The guardrail then errored at request time and the failure surfaced as an unrelated Playwright assertion in presidioUserStory.spec.ts rather than as the missing fixture it actually was. Fail the local runner with the port in the message, and add the matching wait step to both CircleCI UI jobs, which had no readiness check at all. --- .circleci/config.yml | 18 ++++++++++++++++++ tests/e2e/ui/run_e2e.sh | 7 ++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 276555c0cf7..aa851f829e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2652,6 +2652,15 @@ jobs: name: Start mock Presidio server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py background: true + - run: + name: Wait for mock Presidio server + command: | + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi + sleep 1 + done + echo "Mock Presidio server never answered /health on port 8091" >&2 + exit 1 - run: name: Start LiteLLM proxy environment: @@ -2786,6 +2795,15 @@ jobs: name: Start mock Presidio server command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py background: true + - run: + name: Wait for mock Presidio server + command: | + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi + sleep 1 + done + echo "Mock Presidio server never answered /health on port 8091" >&2 + exit 1 - run: name: Start LiteLLM proxy under a server root path environment: diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index ad0454b6347..beb1bc8bf3b 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -212,10 +212,15 @@ echo "=== Starting mock Presidio server ===" uv run --no-sync python "$SCRIPT_DIR/fixtures/mock_presidio_server/server.py" & MOCK_PRESIDIO_PID=$! +PRESIDIO_READY=0 for i in $(seq 1 15); do - if curl -sf http://127.0.0.1:${MOCK_PRESIDIO_PORT}/health >/dev/null 2>&1; then break; fi + if curl -sf http://127.0.0.1:${MOCK_PRESIDIO_PORT}/health >/dev/null 2>&1; then PRESIDIO_READY=1; break; fi sleep 1 done +if [ "$PRESIDIO_READY" -ne 1 ]; then + echo "Mock Presidio server never answered /health on port ${MOCK_PRESIDIO_PORT}" >&2 + exit 1 +fi # --- LiteLLM proxy --- echo "=== Starting LiteLLM proxy ===" From bbc6ca6205c00dcb47a56789b4a639a29ed0e468 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 6 Sep 2026 07:04:21 -0700 Subject: [PATCH 3/5] fix(e2e/ui): stop the masked-prompt assertion tripping strict mode The Logs drawer renders the masked prompt in three places, so matching it without narrowing raised a strict mode violation instead of asserting visibility. CI caught it as a flake: the spec failed its first attempt on b0e53bfbe4 and passed on retry, which is a locator defect rather than a timing one and would have gone red on any run that saw all three nodes. Narrow to the first match, matching the guardrail-name assertion above it. The leak checks below stay on toHaveCount(0), which is unaffected by multiple matches and is what actually proves nothing raw reached the drawer. --- tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts index 975b343287a..3ab5b820896 100644 --- a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts +++ b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts @@ -139,9 +139,8 @@ test.describe("Presidio PII guardrail, end to end from the dashboard", () => { const drawer = page.getByRole("dialog").first(); await expect(drawer.getByText("Guardrails & Policy Compliance")).toBeVisible({ timeout: 20_000 }); await expect(drawer.getByText(`Pre-call guardrail: ${guardrailName}`).first()).toBeVisible({ timeout: 20_000 }); - await expect(drawer.getByText(`${marker}. Email me at or call .`)).toBeVisible({ - timeout: 20_000, - }); + const maskedPrompt = drawer.getByText(`${marker}. Email me at or call .`); + await expect(maskedPrompt.first()).toBeVisible({ timeout: 20_000 }); await drawer.getByText("2 matched").first().click(); await expect(drawer.getByText("Detected Entities (2)").first()).toBeVisible({ timeout: 10_000 }); From 70f1894c28398d8c7ceaace23b876a831af59a25 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 6 Sep 2026 07:21:36 -0700 Subject: [PATCH 4/5] fix(e2e/ui): wait on the visible masked prompt, not the first match The Logs drawer renders the masked prompt in three nodes and the first one in DOM order is hidden, so the previous commit's .first() traded a strict mode violation for a locator that waits 20s on an invisible element. Local runs against a warm stack failed on it every time, resolving the node 22 times and reporting "unexpected value hidden" each time. Use onlyVisible, the helper this spec already uses for the playground selectors, which filters to the visible node before taking the first. Solo runs go 3 for 3 and the guardrails folder passes 5 of 5. Also drop the 30s wait on the PII step added while chasing this: cold start was never the cause, and it left the spec sitting on a dead locator longer. --- tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts index 3ab5b820896..ff4d6260312 100644 --- a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts +++ b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts @@ -139,8 +139,8 @@ test.describe("Presidio PII guardrail, end to end from the dashboard", () => { const drawer = page.getByRole("dialog").first(); await expect(drawer.getByText("Guardrails & Policy Compliance")).toBeVisible({ timeout: 20_000 }); await expect(drawer.getByText(`Pre-call guardrail: ${guardrailName}`).first()).toBeVisible({ timeout: 20_000 }); - const maskedPrompt = drawer.getByText(`${marker}. Email me at or call .`); - await expect(maskedPrompt.first()).toBeVisible({ timeout: 20_000 }); + const maskedPrompt = onlyVisible(drawer.getByText(`${marker}. Email me at or call .`)); + await expect(maskedPrompt).toBeVisible({ timeout: 20_000 }); await drawer.getByText("2 matched").first().click(); await expect(drawer.getByText("Detected Entities (2)").first()).toBeVisible({ timeout: 10_000 }); From fdd1c910963a9161709b1b0eb5faa5e339e7182a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sun, 6 Sep 2026 08:34:19 -0700 Subject: [PATCH 5/5] fix(e2e/ui): scope the drawer assertions to the visible, exact match CI flagged the spec flaky twice more. Both were the same defect in different places: the Logs drawer renders several nodes per string and the first in DOM order is often hidden, so .first() waited 20s on an invisible element. The entity assertions had a second problem on top, since getByText("EMAIL_ADDRESS") substring-matched the masked prompt div, whose text contains , rather than the entity chip. Route every drawer assertion through onlyVisible, and match the entity type and score exactly, which is what the panel renders them as: entity_type and "Score: N.NN" each get their own span. Verified on a live stack: 6 of 6 solo runs and the guardrails folder 5 of 5. Mutating the analyzer to detect nothing turns the spec red on the raw address reaching the spend log, so the assertions still carry their weight. --- .../guardrails/presidioUserStory.spec.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts index ff4d6260312..d4ed5308342 100644 --- a/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts +++ b/tests/e2e/ui/tests/guardrails/presidioUserStory.spec.ts @@ -137,17 +137,17 @@ test.describe("Presidio PII guardrail, end to end from the dashboard", () => { await row.click(); const drawer = page.getByRole("dialog").first(); - await expect(drawer.getByText("Guardrails & Policy Compliance")).toBeVisible({ timeout: 20_000 }); - await expect(drawer.getByText(`Pre-call guardrail: ${guardrailName}`).first()).toBeVisible({ timeout: 20_000 }); - const maskedPrompt = onlyVisible(drawer.getByText(`${marker}. Email me at or call .`)); - await expect(maskedPrompt).toBeVisible({ timeout: 20_000 }); + await expect(onlyVisible(drawer.getByText("Guardrails & Policy Compliance"))).toBeVisible({ timeout: 20_000 }); + await expect(onlyVisible(drawer.getByText(`Pre-call guardrail: ${guardrailName}`))).toBeVisible({ timeout: 20_000 }); + const maskedPrompt = drawer.getByText(`${marker}. Email me at or call .`); + await expect(onlyVisible(maskedPrompt)).toBeVisible({ timeout: 20_000 }); - await drawer.getByText("2 matched").first().click(); - await expect(drawer.getByText("Detected Entities (2)").first()).toBeVisible({ timeout: 10_000 }); - await expect(drawer.getByText("EMAIL_ADDRESS").first()).toBeVisible({ timeout: 10_000 }); - await expect(drawer.getByText("PHONE_NUMBER").first()).toBeVisible({ timeout: 10_000 }); - await expect(drawer.getByText("Score: 1.00").first()).toBeVisible({ timeout: 10_000 }); - await expect(drawer.getByText("Score: 0.75").first()).toBeVisible({ timeout: 10_000 }); + await onlyVisible(drawer.getByText("2 matched")).click(); + await expect(onlyVisible(drawer.getByText("Detected Entities (2)"))).toBeVisible({ timeout: 10_000 }); + await expect(onlyVisible(drawer.getByText("EMAIL_ADDRESS", { exact: true }))).toBeVisible({ timeout: 10_000 }); + await expect(onlyVisible(drawer.getByText("PHONE_NUMBER", { exact: true }))).toBeVisible({ timeout: 10_000 }); + await expect(onlyVisible(drawer.getByText("Score: 1.00", { exact: true }))).toBeVisible({ timeout: 10_000 }); + await expect(onlyVisible(drawer.getByText("Score: 0.75", { exact: true }))).toBeVisible({ timeout: 10_000 }); await expect(drawer.getByText(RAW_EMAIL)).toHaveCount(0); await expect(drawer.getByText(RAW_PHONE)).toHaveCount(0);