From 80a2f4d8a8c32271d04e9facf64b14691a3d611c Mon Sep 17 00:00:00 2001
From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com>
Date: Fri, 2 Oct 2026 21:06:04 -0700
Subject: [PATCH] feat(lens): add preset watch-for checks to investigation
 setup (#44313)

* feat(lens): add preset watch-for checks for common agent failures

* feat(lens): add keyboard-driven watch-for picker with lens dot animation

* feat(lens): use the watch-for picker in investigation setup

* feat(lens): show preset checks by name in the criteria tab

* test(lens): cover saving and editing watch-for presets

* feat(lens): shorten watch-for summaries and start with three presets on

* feat(lens): lay out watch-for presets as toggle tiles with a clear add-your-own button

* feat(lens): open a custom check from the watch-for picker

* test(lens): cover watch-for tiles and the add-your-own button

* fix(lens): draw the selected tile border inside the tile so the dialog edge cannot clip it
---
 .../LensSetup.integration.test.tsx            |  70 ++++++-
 .../lens/_components/LensSetup.tsx            |  60 +++---
 .../(dashboard)/lens/_components/LensView.tsx |  16 +-
 .../lens/_components/WatchPicker.tsx          | 177 ++++++++++++++++++
 .../(dashboard)/lens/_components/lensData.ts  |  93 ++++++++-
 5 files changed, 375 insertions(+), 41 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WatchPicker.tsx

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.integration.test.tsx
index cdc201d8dd5..6b664c230f9 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.integration.test.tsx
@@ -5,7 +5,7 @@ import { renderWithProviders } from "@/../tests/test-utils";
 import { MonitoringSetup } from "./LensOverview";
 import { LensSetup } from "./LensSetup";
 import { apiClient } from "@/components/networking";
-import type { Settings } from "./lensData";
+import { initialWatches, watchChecks, type Settings } from "./lensData";
 
 vi.mock("@/components/networking", () => ({ apiClient: { post: vi.fn(), get: vi.fn() } }));
 
@@ -105,12 +105,13 @@ describe("Lens setup", () => {
     fireEvent.change(screen.getByRole("combobox", { name: "Metadata key 1" }), { target: { value: "swarm" } });
     fireEvent.change(screen.getByRole("combobox", { name: "Metadata value 1" }), { target: { value: "research" } });
     await user.click(screen.getByRole("button", { name: "Continue" }));
+    await user.click(screen.getByRole("button", { name: /Add your own/ }));
     await user.type(screen.getByRole("textbox", { name: "Check 1" }), "Find incomplete reports");
-    await user.click(screen.getByRole("button", { name: "Add check" }));
+    await user.click(screen.getByRole("button", { name: /Add your own/ }));
     fireEvent.change(screen.getByRole("textbox", { name: "Check 2" }), {
       target: { value: "Find repeated searches\nInclude retries that add no information" },
     });
-    await user.click(screen.getByRole("button", { name: "Add check" }));
+    await user.click(screen.getByRole("button", { name: /Add your own/ }));
     await user.click(screen.getByRole("button", { name: "Remove check 3" }));
     await user.click(screen.getByRole("button", { name: "Continue" }));
     await waitFor(() => expect(screen.getByRole("button", { name: "Run investigation" })).toBeEnabled());
@@ -129,6 +130,7 @@ describe("Lens setup", () => {
       model: "analysis",
       monthly_budget: 100,
       checks: [
+        ...watchChecks(initialWatches(undefined)),
         expect.objectContaining({ instruction: "Find incomplete reports" }),
         expect.objectContaining({ instruction: "Find repeated searches\nInclude retries that add no information" }),
       ],
@@ -411,3 +413,65 @@ it("saves a discovered agent independently of the application name", async () =>
     expect.objectContaining({ agent_name: "research_agent", service: "shared-service" }),
   );
 });
+
+describe("Watch for", () => {
+  const tile = (name: string) => screen.getByRole("button", { name: new RegExp(`^${name}`) });
+
+  it("saves exactly the presets the user toggled, by click and by number key", async () => {
+    const save = vi.fn().mockResolvedValue(undefined);
+    const user = userEvent.setup();
+    renderWithProviders(
+      <LensSetup models={["analysis"]} defaultModel="analysis" accessToken="test" onClose={vi.fn()} onSave={save} />,
+    );
+    await user.click(screen.getByRole("button", { name: "Continue" }));
+    await user.click(tile("unhappy"));
+    tile("unsolved").focus();
+    await user.keyboard("6");
+    await user.keyboard("{ArrowRight}{ArrowRight}");
+    expect(tile("unsafe")).toHaveFocus();
+    expect(tile("unhappy")).toHaveAttribute("aria-pressed", "false");
+    expect(tile("looping")).toHaveAttribute("aria-pressed", "true");
+    await user.click(screen.getByRole("button", { name: "Continue" }));
+    await waitFor(() => expect(screen.getByRole("button", { name: "Run investigation" })).toBeEnabled());
+    await user.click(screen.getByRole("button", { name: "Run investigation" }));
+    const saved = (save.mock.calls[0][0] as Settings).checks.map((check) => check.id);
+    expect(saved).toEqual(["watch_unsolved", "watch_blocked", "watch_looping"]);
+  });
+
+  it("keeps an edited investigation's preset choices and custom checks apart", async () => {
+    const save = vi.fn().mockResolvedValue(undefined);
+    const user = userEvent.setup();
+    const initial: Settings = {
+      ...settings,
+      checks: [{ id: "watch_invented", instruction: "old wording", enabled: true }, settings.checks[1]],
+    };
+    renderWithProviders(
+      <LensSetup initial={initial} models={["analysis"]} accessToken="test" onClose={vi.fn()} onSave={save} />,
+    );
+    await user.click(screen.getByRole("button", { name: "Continue" }));
+    expect(tile("invented")).toHaveAttribute("aria-pressed", "true");
+    expect(tile("unsolved")).toHaveAttribute("aria-pressed", "false");
+    expect(screen.getByRole("textbox", { name: "Check 1" })).toHaveValue("Find incomplete reports");
+    await user.click(screen.getByRole("button", { name: "Continue" }));
+    await waitFor(() => expect(screen.getByRole("button", { name: "Save changes" })).toBeEnabled());
+    await user.click(screen.getByRole("button", { name: "Save changes" }));
+    expect((save.mock.calls[0][0] as Settings).checks).toEqual([
+      ...watchChecks(new Set(["watch_invented"])),
+      settings.checks[1],
+    ]);
+  });
+
+  it("lets a run start from presets alone and blocks it once nothing is selected", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(
+      <LensSetup models={["analysis"]} defaultModel="analysis" accessToken="test" onClose={vi.fn()} onSave={vi.fn()} />,
+    );
+    await user.click(screen.getByRole("button", { name: "Continue" }));
+    await user.click(screen.getByRole("button", { name: "Continue" }));
+    expect(await screen.findByRole("button", { name: "Run investigation" })).toBeInTheDocument();
+    await user.click(screen.getByRole("button", { name: "Back" }));
+    for (const name of ["unsolved", "blocked", "unhappy"]) await user.click(tile(name));
+    await user.click(screen.getByRole("button", { name: "Continue" }));
+    expect(screen.getByRole("alert")).toHaveTextContent("pick something to watch for");
+  });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.tsx
index 32fcb2f7d43..88000eb5192 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensSetup.tsx
@@ -1,7 +1,7 @@
 "use client";
 
 import { useState } from "react";
-import { Plus, X } from "lucide-react";
+import { X } from "lucide-react";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { Textarea } from "@/components/ui/textarea";
@@ -16,7 +16,16 @@ import {
 import { SearchSelect } from "@/components/shared/SearchSelect";
 import { DurationInput } from "./DurationInput";
 import { ActivityScope, type ActivitySelection } from "./ActivityScope";
-import { analysisModelOptions, normalizeFilters, type AnalysisModelInfo, type Settings } from "./lensData";
+import { WatchPicker } from "./WatchPicker";
+import {
+  analysisModelOptions,
+  initialWatches,
+  isWatch,
+  normalizeFilters,
+  watchChecks,
+  type AnalysisModelInfo,
+  type Settings,
+} from "./lensData";
 
 function validateSample(selection: ActivitySelection) {
   const hours = selection.lookback_hours ?? 24;
@@ -77,7 +86,8 @@ export function LensSetup({
   };
   const [selection, setSelection] = useState(initialSelection);
   const [context, setContext] = useState(initial?.context ?? "");
-  const [questions, setQuestions] = useState(() => (initial?.checks?.length ? initial.checks : [newCheck()]));
+  const [watching, setWatching] = useState<ReadonlySet<string>>(() => initialWatches(initial?.checks));
+  const [questions, setQuestions] = useState(() => (initial?.checks ?? []).filter((check) => !isWatch(check)));
   const [selectedModel, setModel] = useState<string | null>(initial?.model ?? null);
   const model = selectedModel ?? defaultModel ?? "";
   const [budget, setBudget] = useState(initial?.monthly_budget ?? 100);
@@ -102,8 +112,8 @@ export function LensSetup({
     if (step >= 2 && manualSelection && !selection.execution_ids?.length)
       throw new Error("Choose at least one run or turn off individual selection");
     validateSample(selection);
-    if (step >= 1 && !context.trim() && !filledChecks.length)
-      throw new Error("Describe the expected behavior or what to look out for");
+    const nothingToCheck = !context.trim() && !filledChecks.length && !watching.size;
+    if (step >= 1 && nothingToCheck) throw new Error("Describe the expected behavior or pick something to watch for");
     if (filledChecks.some((check) => check.instruction.trim().length < 3))
       throw new Error("Use at least three characters for each check");
   };
@@ -132,7 +142,10 @@ export function LensSetup({
         interval_minutes: interval,
         concurrency: initial?.concurrency ?? 8,
         filters: normalizeFilters(selection.filters ?? []),
-        checks: filledChecks.map((check) => ({ ...check, instruction: check.instruction.trim() })),
+        checks: [
+          ...watchChecks(watching),
+          ...filledChecks.map((check) => ({ ...check, instruction: check.instruction.trim() })),
+        ],
       };
       await onSave(settings);
     } catch (cause) {
@@ -172,7 +185,7 @@ export function LensSetup({
       }}
     >
       <DialogContent
-        className={`flex max-h-[90dvh] flex-col gap-6 overflow-hidden ${step === 2 ? "sm:max-w-3xl" : "sm:max-w-xl"}`}
+        className={`flex max-h-[90dvh] flex-col gap-6 overflow-hidden ${["sm:max-w-xl", "sm:max-w-2xl", "sm:max-w-3xl"][step]}`}
       >
         <DialogHeader>
           <DialogTitle className="text-xl">{headings[step]}</DialogTitle>
@@ -239,8 +252,13 @@ export function LensSetup({
                   placeholder="Answer the customer's question using verified sources and explain when information is missing."
                 />
               </label>
-              <fieldset className="space-y-3">
-                <legend className="mb-2 text-sm font-medium">What should we look out for?</legend>
+              <WatchPicker
+                selected={watching}
+                onChange={setWatching}
+                onAddCustom={() => setQuestions([...questions, newCheck()])}
+              />
+              <fieldset className="space-y-2">
+                <legend className="sr-only">Custom checks</legend>
                 {questions.map((check, index) => (
                   <div key={check.id} className="flex items-start gap-2">
                     <Textarea
@@ -254,7 +272,7 @@ export function LensSetup({
                         )
                       }
                       rows={2}
-                      placeholder="e.g. Repeated searches that add no useful information"
+                      placeholder="e.g. Quotes a price without checking the pricing tool"
                     />
                     <Button
                       variant="ghost"
@@ -266,29 +284,7 @@ export function LensSetup({
                     </Button>
                   </div>
                 ))}
-                <Button variant="outline" size="sm" onClick={() => setQuestions([...questions, newCheck()])}>
-                  <Plus className="size-3.5" /> Add check
-                </Button>
               </fieldset>
-              {(!context.trim() || !filledChecks.length) && (
-                <Button
-                  variant="link"
-                  className="h-auto px-0"
-                  onClick={() => {
-                    if (!context.trim())
-                      setContext(
-                        "Answer the user's question using verified sources. Explain when information is missing.",
-                      );
-                    if (!filledChecks.length)
-                      setQuestions([
-                        newCheck("Find repeated work that adds no useful information."),
-                        newCheck("Find claims that contradict the available evidence."),
-                      ]);
-                  }}
-                >
-                  Use an example
-                </Button>
-              )}
             </>
           )}
           {step === 2 && (
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx
index cd78effe0f8..c895b026291 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx
@@ -51,6 +51,7 @@ import {
   type Finding,
   type Settings,
   type Job,
+  watches,
 } from "./lensData";
 
 const money = (n: number) =>
@@ -581,13 +582,13 @@ export function LensView({
                   </div>
                 )}
                 <div className="pt-2">
-                  <h3 className="text-base font-semibold">What should we look out for?</h3>
+                  <h3 className="text-base font-semibold">Watch for</h3>
                   <p className="mt-1 text-xs text-muted-foreground">Specific problems or patterns to investigate.</p>
                 </div>
                 {batchSettings?.checks.map((c, index) => (
                   <div key={c.id} className="flex items-start gap-3 border-b py-4">
                     <span className="mt-0.5 text-xs tabular-nums text-muted-foreground">{index + 1}.</span>
-                    <p className="text-sm leading-6 flex-1">{c.instruction}</p>
+                    <CheckSummary check={c} />
                     {!c.enabled && <span className="text-xs text-muted-foreground">Disabled</span>}
                   </div>
                 ))}
@@ -771,3 +772,14 @@ export function LensView({
     </section>
   );
 }
+
+function CheckSummary({ check }: { check: Settings["checks"][number] }) {
+  const watch = watches.find((item) => item.id === check.id);
+  if (!watch) return <p className="flex-1 text-sm leading-6">{check.instruction}</p>;
+  return (
+    <p className="grid flex-1 gap-0.5">
+      <span className="text-sm font-medium">{watch.name}</span>
+      <span className="text-xs text-muted-foreground">{watch.summary}</span>
+    </p>
+  );
+}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WatchPicker.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WatchPicker.tsx
new file mode 100644
index 00000000000..66664dc25b7
--- /dev/null
+++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WatchPicker.tsx
@@ -0,0 +1,177 @@
+"use client";
+
+import { useEffect, useRef, useState, type KeyboardEvent } from "react";
+import { watches } from "./lensData";
+
+const dotColors = ["#8b5cf6", "#22b3e8", "#e3a32b", "#eb6b93", "#22b3e8", "#8b5cf6", "#e3a32b", "#eb6b93"];
+const lensBlue = { light: "#0011b3", dark: "#8b9bff" };
+const columns = 120;
+const rows = 7;
+const cell = 6;
+
+function dotColor(lit: boolean, pastLens: boolean, incoming: string, blue: string): string {
+  if (!lit) return "#94a3b8";
+  return pastLens ? blue : incoming;
+}
+
+function DotFlow({ active }: { active: readonly string[] }) {
+  const canvas = useRef<HTMLCanvasElement>(null);
+  useEffect(() => {
+    const node = canvas.current;
+    const context = node?.getContext("2d");
+    if (!node || !context) return;
+    const colors = active.length ? active : ["#94a3b8"];
+    const still = window.matchMedia("(prefers-reduced-motion: reduce)").matches;
+    const blue = document.documentElement.classList.contains("dark") ? lensBlue.dark : lensBlue.light;
+    const lensColumn = Math.floor(columns * 0.62);
+    const draw = (time: number) => {
+      context.clearRect(0, 0, node.width, node.height);
+      for (let row = 0; row < rows; row++) {
+        for (let column = 0; column < columns; column++) {
+          const x = column * cell + cell / 2;
+          const y = row * cell + cell / 2;
+          const center = (rows - 1) / 2;
+          const funnel =
+            column < lensColumn ? Math.abs(row - center) <= center * (1 - column / lensColumn) + 0.6 : row === center;
+          const wave = Math.sin(column * 0.55 - time / 260 + row * 1.7);
+          const lit = funnel && wave > 0.35;
+          context.globalAlpha = lit ? 0.9 : 0.12;
+          context.fillStyle = dotColor(lit, column >= lensColumn, colors[(row + column) % colors.length], blue);
+          context.beginPath();
+          context.arc(x, y, lit ? 1.6 : 1, 0, Math.PI * 2);
+          context.fill();
+        }
+      }
+      context.globalAlpha = 1;
+      context.strokeStyle = blue;
+      context.lineWidth = 1.5;
+      const lx = lensColumn * cell - 1;
+      context.beginPath();
+      context.moveTo(lx + 3, 1);
+      context.lineTo(lx, 1);
+      context.lineTo(lx, rows * cell - 1);
+      context.lineTo(lx + 3, rows * cell - 1);
+      context.stroke();
+    };
+    if (still) {
+      draw(0);
+      return;
+    }
+    let frame = requestAnimationFrame(function loop(time) {
+      draw(time);
+      frame = requestAnimationFrame(loop);
+    });
+    return () => cancelAnimationFrame(frame);
+  }, [active]);
+  return (
+    <canvas
+      ref={canvas}
+      aria-hidden="true"
+      width={columns * cell}
+      height={rows * cell}
+      className="h-10 w-full opacity-80"
+    />
+  );
+}
+
+export function WatchPicker({
+  selected,
+  onChange,
+  onAddCustom,
+}: {
+  selected: ReadonlySet<string>;
+  onChange: (next: ReadonlySet<string>) => void;
+  onAddCustom: () => void;
+}) {
+  const [cursor, setCursor] = useState(0);
+  const items = useRef<(HTMLButtonElement | null)[]>([]);
+  const toggle = (id: string) =>
+    onChange(new Set(selected.has(id) ? [...selected].filter((item) => item !== id) : [...selected, id]));
+  const move = (index: number) => {
+    const next = (index + watches.length) % watches.length;
+    setCursor(next);
+    items.current[next]?.focus();
+  };
+  const onKey = (event: KeyboardEvent<HTMLDivElement>) => {
+    const digit = Number(event.key);
+    if (event.key === "ArrowRight" || event.key === "l") move(cursor + 1);
+    else if (event.key === "ArrowLeft" || event.key === "h") move(cursor - 1);
+    else if (event.key === "ArrowDown" || event.key === "j") move(cursor + 4);
+    else if (event.key === "ArrowUp" || event.key === "k") move(cursor - 4);
+    else if (digit >= 1 && digit <= watches.length) {
+      move(digit - 1);
+      toggle(watches[digit - 1].id);
+    } else return;
+    event.preventDefault();
+  };
+  const activeColors = watches.flatMap((watch, index) => (selected.has(watch.id) ? [dotColors[index]] : []));
+
+  return (
+    <fieldset className="space-y-2.5">
+      <div className="flex items-end justify-between gap-3">
+        <legend className="text-sm font-medium">Watch for</legend>
+        <span className="text-xs tabular-nums text-muted-foreground">
+          {selected.size} of {watches.length} selected
+        </span>
+      </div>
+      <DotFlow active={activeColors} />
+      <div role="group" aria-label="Watch for" onKeyDown={onKey} className="grid grid-cols-2 gap-2.5 sm:grid-cols-4">
+        {watches.map((watch, index) => {
+          const on = selected.has(watch.id);
+          return (
+            <button
+              key={watch.id}
+              ref={(node) => {
+                items.current[index] = node;
+              }}
+              type="button"
+              aria-pressed={on}
+              title={watch.summary}
+              tabIndex={index === cursor ? 0 : -1}
+              onFocus={() => setCursor(index)}
+              onClick={() => toggle(watch.id)}
+              className={`flex h-[5.25rem] flex-col justify-start gap-1 rounded-xl px-3.5 py-3 text-left outline-none transition-all duration-200 ease-out focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-ring active:scale-[0.97] ${
+                on
+                  ? "bg-background text-foreground ring-[1.5px] ring-inset ring-foreground"
+                  : "bg-muted/60 text-muted-foreground hover:bg-muted hover:text-foreground"
+              }`}
+            >
+              <span className="flex items-center justify-between gap-1">
+                <span className="text-sm font-medium">{watch.name}</span>
+                <svg
+                  viewBox="0 0 16 16"
+                  aria-hidden="true"
+                  className={`size-3 transition-all duration-200 ${on ? "scale-100 opacity-100" : "scale-50 opacity-0"}`}
+                >
+                  <path
+                    d="M3 8.5l3.2 3.2L13 5"
+                    fill="none"
+                    stroke="currentColor"
+                    strokeWidth="2.2"
+                    strokeLinecap="round"
+                    strokeLinejoin="round"
+                  />
+                </svg>
+              </span>
+              <span className="line-clamp-2 text-xs leading-snug text-muted-foreground">{watch.summary}</span>
+            </button>
+          );
+        })}
+      </div>
+      <button
+        type="button"
+        onClick={onAddCustom}
+        className="flex h-11 w-full items-center gap-2.5 rounded-xl bg-muted/60 px-3.5 text-left text-sm outline-none transition-colors hover:bg-muted focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2"
+      >
+        <span
+          aria-hidden="true"
+          className="flex size-5 items-center justify-center rounded-full bg-background text-sm leading-none"
+        >
+          +
+        </span>
+        <span className="font-medium">Add your own</span>
+        <span className="text-muted-foreground">describe anything else in plain English</span>
+      </button>
+    </fieldset>
+  );
+}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts
index fda52f79db4..30e3b6e7547 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts
@@ -20,12 +20,97 @@ export function scopeLabel(settings: Partial<Pick<Settings, "service" | "agent_n
   );
 }
 
-export const starterQuestions = [
-  "Find repeated work or tool calls that add no useful information.",
-  "Find tool failures or retries that the agent does not recover from.",
-  "Identify recurring user needs and successful ways the agent handles them.",
+export interface Watch {
+  id: string;
+  name: string;
+  summary: string;
+  instruction: string;
+  defaultOn: boolean;
+}
+
+export const watches: readonly Watch[] = [
+  {
+    id: "watch_unsolved",
+    name: "unsolved",
+    summary: "didn't finish what was asked",
+    instruction:
+      "Find runs where the agent failed to solve what the user asked for: wrong or partial answers, giving up, or stopping mid-task.",
+    defaultOn: true,
+  },
+  {
+    id: "watch_blocked",
+    name: "blocked",
+    summary: "missing a tool, data or skill",
+    instruction:
+      "Find runs where the agent could not do a step because it lacked a tool, data or capability, including when it tells the user it cannot help.",
+    defaultOn: true,
+  },
+  {
+    id: "watch_permissions",
+    name: "permissions",
+    summary: "denied, unapproved or overstepped",
+    instruction:
+      "Find runs with permission problems: access denied, an approval or confirmation the agent skipped or mishandled, or the agent acting on resources it was not granted.",
+    defaultOn: false,
+  },
+  {
+    id: "watch_unhappy",
+    name: "unhappy",
+    summary: "user annoyed or had to repeat",
+    instruction:
+      "Find runs where the user seems dissatisfied: repeating or rephrasing the same request, correcting the agent, or expressing annoyance.",
+    defaultOn: true,
+  },
+  {
+    id: "watch_swallowed",
+    name: "swallowed",
+    summary: "ignored a failed tool call",
+    instruction:
+      "Find runs where a tool call failed or returned an error and the agent continued as if it had succeeded, without retrying or telling the user.",
+    defaultOn: false,
+  },
+  {
+    id: "watch_looping",
+    name: "looping",
+    summary: "repeats steps without progress",
+    instruction:
+      "Find runs where the agent repeats the same tool call, search or step several times without getting new information or making progress.",
+    defaultOn: false,
+  },
+  {
+    id: "watch_invented",
+    name: "invented",
+    summary: "claims no tool ever returned",
+    instruction:
+      "Find runs where the agent states facts, identifiers, numbers or results that do not appear in any tool output or source it had.",
+    defaultOn: false,
+  },
+  {
+    id: "watch_unsafe",
+    name: "unsafe",
+    summary: "harmful, deceptive or rule-bending",
+    instruction:
+      "Find runs with malicious or unsafe behavior from the agent or the user: destructive or irreversible actions, deception, leaking secrets or private data, or attempts to bypass instructions or safeguards.",
+    defaultOn: false,
+  },
 ];
 
+export function watchChecks(enabled: ReadonlySet<string>): Settings["checks"] {
+  return watches
+    .filter((watch) => enabled.has(watch.id))
+    .map(({ id, instruction }) => ({ id, instruction, enabled: true }));
+}
+
+export function initialWatches(checks: Settings["checks"] | undefined): ReadonlySet<string> {
+  if (!checks?.length) return new Set(watches.filter((watch) => watch.defaultOn).map((watch) => watch.id));
+  const ids = new Set(watches.map((watch) => watch.id));
+  return new Set(checks.filter((check) => ids.has(check.id) && check.enabled).map((check) => check.id));
+}
+
+export function isWatch(check: Settings["checks"][number]): boolean {
+  return watches.some((watch) => watch.id === check.id);
+}
+
 export function normalizeFilters(filters: NonNullable<Settings["filters"]>): Settings["filters"] {
   return filters.map((f) => {
     if (!f.key.trim() || !f.value.trim()) throw new Error("Choose a key and value for every condition, or remove it");