test(e2e/ui): cover creating, testing and deleting a guardrail (#39053)

* test(e2e/ui): cover creating, testing and deleting a guardrail

The Guardrails page had no browser coverage. The RC checklist covers it by
hand against a live Presidio, which is why it has always been skipped in CI.

These drive the LiteLLM content filter instead, which runs inside the proxy,
so the whole flow is exercised without a third-party moderation service. The
create test does not stop at the table row: it sends a prompt carrying the
keyword it just banned and asserts the gateway refuses it, then sends a clean
prompt through the same guardrail and asserts it is served.

* test(e2e/ui): delete the guardrails these tests create

Review caught the fixtures being left behind. Guardrails are database rows
that show up in the table and in the playground's list, so a run that leaves
them changes what the next run sees.

Also trims the comments that restated what the helpers already say.

* test(e2e/ui): fail the run when guardrail teardown does not delete

Review caught the afterEach discarding the DELETE response, so a failed
cleanup finished quietly and left the guardrail for the next run to trip on.

* test(e2e/ui): wait for a new guardrail to reach the request path

The wizard test drove one chat completion immediately after creating the
guardrail and required a 400. A trace from the deployed stack shows the
record is stored correctly (blocked_words, action BLOCK, block_on_violation)
and the call six seconds later is still served unguarded, so the first
request can land before the proxy picks the guardrail up.

Polls the same call to the same 400 instead, which keeps the assertion and
lets the refresh land. If it never blocks, this stays red, which is what we
want it to say.

---------

Co-authored-by: Claude <noreply@anthropic.com>

tests/e2e/ui/tests/guardrails/guardrails.spec.ts

@@ -1,11 +1,213 @@
-import { test, expect } from "@playwright/test";
+import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
 import { ADMIN_STORAGE_PATH, E2E_TEAM_NO_ADMIN_ID } from "../../constants";
 import { Page } from "../../fixtures/pages";
 import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
+import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic";
+
+interface StoredGuardrail {
+  guardrail_id: string;
+  guardrail_name: string | null;
+}
+
+async function listGuardrails(page: PlaywrightPage): Promise<StoredGuardrail[]> {
+  const res = await page.request.get("/v2/guardrails/list", {
+    headers: { Authorization: `Bearer ${masterKey()}` },
+  });
+  expect(res.ok(), `GET /v2/guardrails/list (${res.status()})`).toBe(true);
+  return ((await res.json()) as { guardrails: StoredGuardrail[] }).guardrails;
+}
+
+async function findGuardrail(page: PlaywrightPage, name: string): Promise<StoredGuardrail | undefined> {
+  return (await listGuardrails(page)).find((row) => row.guardrail_name === name);
+}
+
+const createdGuardrails: string[] = [];
+
+async function createKeywordGuardrailViaApi(page: PlaywrightPage, name: string, keyword: string): Promise<string> {
+  const res = await page.request.post("/guardrails", {
+    headers: { Authorization: `Bearer ${masterKey()}` },
+    data: {
+      guardrail: {
+        guardrail_name: name,
+        litellm_params: {
+          guardrail: "litellm_content_filter",
+          mode: "pre_call",
+          default_on: false,
+          blocked_words: [{ keyword, action: "BLOCK" }],
+        },
+      },
+    },
+  });
+  expect(res.ok(), `POST /guardrails failed (${res.status()}): ${await res.text()}`).toBe(true);
+  createdGuardrails.push(name);
+  const guardrail = await findGuardrail(page, name);
+  expect(guardrail?.guardrail_id, `guardrail ${name} has an id`).toBeTruthy();
+  return guardrail!.guardrail_id;
+}
+
+async function openKeywordsStep(page: PlaywrightPage, name: string) {
+  await page.getByRole("button", { name: "Add New Guardrail" }).click();
+  await page.getByRole("menuitem", { name: "Add Provider Guardrail" }).click();
+
+  const wizard = page.getByRole("dialog", { name: "Create guardrail" });
+  await expect(wizard).toBeVisible({ timeout: 10_000 });
+
+  await wizard.getByRole("textbox", { name: "Guardrail Name" }).fill(name);
+  await wizard.getByRole("combobox", { name: "Guardrail Provider" }).click();
+  // The content filter runs inside the proxy, so this is the one provider a test can
+  // configure end to end without standing up a third-party moderation service.
+  await page.getByRole("option", { name: /LiteLLM Content Filter/ }).click();
+
+  for (const step of ["Topics", "Patterns", "Keywords"]) {
+    await wizard.getByRole("button", { name: "Next" }).click();
+    await expect(wizard).toContainText(step, { timeout: 10_000 });
+  }
+  return wizard;
+}
 
 test.describe("Guardrails", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
+  test.afterEach(async ({ page }) => {
+    // Guardrails live in the database and show up in the table and the playground list, so a run
+    // that leaves them behind changes what the next run sees.
+    for (const name of createdGuardrails.splice(0)) {
+      const guardrail = await findGuardrail(page, name);
+      if (guardrail) {
+        const deleted = await page.request.delete(`/guardrails/${guardrail.guardrail_id}`, {
+          headers: { Authorization: `Bearer ${masterKey()}` },
+        });
+        expect(deleted.ok(), `DELETE /guardrails/${guardrail.guardrail_id} (${deleted.status()})`).toBe(true);
+      }
+    }
+  });
+
+  test("A guardrail created through the wizard blocks the keyword it was given", async ({ page }) => {
+    const stamp = Date.now();
+    const guardrailName = `e2e-guardrail-create-${stamp}`;
+    // Unique per run so a concurrent test's prompt can never trip this guardrail, or vice versa.
+    const bannedKeyword = `e2ebanned${stamp}`;
+
+    await navigateToPage(page, Page.Guardrails);
+    await dismissFeedbackPopup(page);
+
+    createdGuardrails.push(guardrailName);
+    const wizard = await openKeywordsStep(page, guardrailName);
+
+    await wizard.getByRole("button", { name: "Add keyword" }).click();
+    const keywordModal = page.getByRole("dialog", { name: "Add blocked keyword" });
+    await expect(keywordModal).toBeVisible({ timeout: 10_000 });
+    await keywordModal.getByPlaceholder("Enter sensitive keyword or phrase").fill(bannedKeyword);
+    await keywordModal.getByRole("button", { name: "Add", exact: true }).click();
+    await expect(keywordModal).not.toBeVisible({ timeout: 10_000 });
+
+    await wizard.getByRole("button", { name: "Next" }).click();
+    await wizard.getByRole("button", { name: "Create Guardrail" }).click();
+    await expect(wizard).not.toBeVisible({ timeout: 15_000 });
+
+    await expect(page.getByRole("row").filter({ hasText: guardrailName })).toBeVisible({ timeout: 15_000 });
+    expect(await findGuardrail(page, guardrailName), "guardrail readable from /v2/guardrails/list").toBeTruthy();
+
+    // A row in the table only proves the record was written. The point of a guardrail is that it
+    // refuses traffic, so drive a request through it.
+    //
+    // Polled: a guardrail written through /guardrails reaches the request path on the proxy's
+    // periodic refresh, so the first call after creation can still be served unguarded. The
+    // assertion is unchanged, it just allows that refresh to land.
+    let blockedBody = "";
+    await expect
+      .poll(
+        async () => {
+          const res = await page.request.post("/v1/chat/completions", {
+            headers: { Authorization: `Bearer ${masterKey()}` },
+            data: {
+              model: CHAT_MODEL_A,
+              messages: [{ role: "user", content: `please tell me about ${bannedKeyword}` }],
+              guardrails: [guardrailName],
+            },
+          });
+          blockedBody = await res.text();
+          return res.status();
+        },
+        { message: "a prompt carrying the banned keyword is refused", timeout: 60_000 },
+      )
+      .toBe(400);
+    expect(blockedBody).toContain(bannedKeyword);
+
+    const allowed = await page.request.post("/v1/chat/completions", {
+      headers: { Authorization: `Bearer ${masterKey()}` },
+      data: {
+        model: CHAT_MODEL_A,
+        messages: [{ role: "user", content: "hello there" }],
+        guardrails: [guardrailName],
+      },
+    });
+    expect(allowed.status(), "a clean prompt still gets through the same guardrail").toBe(200);
+    expect((await allowed.json()).choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT);
+  });
+
+  test("The Test Playground reports the verdict for the text it is given", async ({ page }) => {
+    const stamp = Date.now();
+    const guardrailName = `e2e-guardrail-play-${stamp}`;
+    const bannedKeyword = `e2eplay${stamp}`;
+    await createKeywordGuardrailViaApi(page, guardrailName, bannedKeyword);
+
+    await navigateToPage(page, Page.Guardrails);
+    await dismissFeedbackPopup(page);
+
+    await page.getByRole("tab", { name: "Test Playground" }).click();
+    // Every tab on this page stays mounted, so the other tabs' search boxes match too.
+    const playground = page.getByRole("tabpanel", { name: "Test Playground" });
+    await playground.getByPlaceholder("Search guardrails...").fill(guardrailName);
+    await playground.getByText(guardrailName, { exact: true }).click();
+
+    const input = playground.getByPlaceholder("Enter text to test with guardrails...");
+    await input.fill(`this sentence contains ${bannedKeyword}`);
+    await playground.getByRole("button", { name: /^Test 1 guardrail$/ }).click();
+
+    // The playground is where an admin checks a guardrail before rolling it out, so the
+    // verdict it prints has to be the one the gateway would give.
+    await expect(playground.getByText(`${guardrailName} - Error`)).toBeVisible({ timeout: 20_000 });
+    await expect(playground.getByText(new RegExp(`Content blocked.*${bannedKeyword}`))).toBeVisible({
+      timeout: 10_000,
+    });
+
+    await input.fill("this sentence is perfectly ordinary");
+    await playground.getByRole("button", { name: /^Test 1 guardrail$/ }).click();
+
+    await expect(playground.getByText(`${guardrailName} - Error`)).toHaveCount(0, { timeout: 20_000 });
+    await expect(playground.getByText("this sentence is perfectly ordinary").last()).toBeVisible({ timeout: 10_000 });
+  });
+
+  test("Delete a guardrail", async ({ page }) => {
+    const stamp = Date.now();
+    const guardrailName = `e2e-guardrail-delete-${stamp}`;
+    const guardrailId = await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`);
+
+    await navigateToPage(page, Page.Guardrails);
+    await dismissFeedbackPopup(page);
+
+    await expect(page.getByRole("row").filter({ hasText: guardrailName })).toBeVisible({ timeout: 15_000 });
+
+    await page.getByTestId(`guardrail-actions-${guardrailId}`).click();
+    await page.getByTestId("guardrail-action-delete").click();
+
+    const modal = page.getByRole("dialog");
+    await expect(modal).toBeVisible({ timeout: 5_000 });
+    await modal.getByRole("button", { name: "Delete", exact: true }).click();
+
+    await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(0, { timeout: 15_000 });
+
+    // The RC checklist deletes then reloads, because a row vanishing from the table has
+    // fooled us before; assert against the route the reload would read.
+    await expect
+      .poll(async () => await findGuardrail(page, guardrailName), {
+        message: `guardrail ${guardrailName} still listed after delete`,
+        timeout: 15_000,
+      })
+      .toBeUndefined();
+  });
+
   test("Create a Presidio guardrail, see it in team settings, and delete it", async ({ page }) => {
     const guardrailName = `e2e-presidio-${Date.now()}`;