mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
feat(lens): add preset watch-for checks to investigation setup (#44313)
* feat(lens): add preset watch-for checks for common agent failures * feat(lens): add keyboard-driven watch-for picker with lens dot animation * feat(lens): use the watch-for picker in investigation setup * feat(lens): show preset checks by name in the criteria tab * test(lens): cover saving and editing watch-for presets * feat(lens): shorten watch-for summaries and start with three presets on * feat(lens): lay out watch-for presets as toggle tiles with a clear add-your-own button * feat(lens): open a custom check from the watch-for picker * test(lens): cover watch-for tiles and the add-your-own button * fix(lens): draw the selected tile border inside the tile so the dialog edge cannot clip it
This commit is contained in:
ishaan-berri
GitHub
parent
54260bec88
commit
80a2f4d8a8
5 changed files with 375 additions and 41 deletions
|
|
@ -5,7 +5,7 @@ import { renderWithProviders } from "@/../tests/test-utils";
|
|||
import { MonitoringSetup } from "./LensOverview";
|
||||
import { LensSetup } from "./LensSetup";
|
||||
import { apiClient } from "@/components/networking";
|
||||
import type { Settings } from "./lensData";
|
||||
import { initialWatches, watchChecks, type Settings } from "./lensData";
|
||||
|
||||
vi.mock("@/components/networking", () => ({ apiClient: { post: vi.fn(), get: vi.fn() } }));
|
||||
|
||||
|
|
@ -105,12 +105,13 @@ describe("Lens setup", () => {
|
|||
fireEvent.change(screen.getByRole("combobox", { name: "Metadata key 1" }), { target: { value: "swarm" } });
|
||||
fireEvent.change(screen.getByRole("combobox", { name: "Metadata value 1" }), { target: { value: "research" } });
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
await user.click(screen.getByRole("button", { name: /Add your own/ }));
|
||||
await user.type(screen.getByRole("textbox", { name: "Check 1" }), "Find incomplete reports");
|
||||
await user.click(screen.getByRole("button", { name: "Add check" }));
|
||||
await user.click(screen.getByRole("button", { name: /Add your own/ }));
|
||||
fireEvent.change(screen.getByRole("textbox", { name: "Check 2" }), {
|
||||
target: { value: "Find repeated searches\nInclude retries that add no information" },
|
||||
});
|
||||
await user.click(screen.getByRole("button", { name: "Add check" }));
|
||||
await user.click(screen.getByRole("button", { name: /Add your own/ }));
|
||||
await user.click(screen.getByRole("button", { name: "Remove check 3" }));
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Run investigation" })).toBeEnabled());
|
||||
|
|
@ -129,6 +130,7 @@ describe("Lens setup", () => {
|
|||
model: "analysis",
|
||||
monthly_budget: 100,
|
||||
checks: [
|
||||
...watchChecks(initialWatches(undefined)),
|
||||
expect.objectContaining({ instruction: "Find incomplete reports" }),
|
||||
expect.objectContaining({ instruction: "Find repeated searches\nInclude retries that add no information" }),
|
||||
],
|
||||
|
|
@ -411,3 +413,65 @@ it("saves a discovered agent independently of the application name", async () =>
|
|||
expect.objectContaining({ agent_name: "research_agent", service: "shared-service" }),
|
||||
);
|
||||
});
|
||||
|
||||
describe("Watch for", () => {
|
||||
const tile = (name: string) => screen.getByRole("button", { name: new RegExp(`^${name}`) });
|
||||
|
||||
it("saves exactly the presets the user toggled, by click and by number key", async () => {
|
||||
const save = vi.fn().mockResolvedValue(undefined);
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<LensSetup models={["analysis"]} defaultModel="analysis" accessToken="test" onClose={vi.fn()} onSave={save} />,
|
||||
);
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
await user.click(tile("unhappy"));
|
||||
tile("unsolved").focus();
|
||||
await user.keyboard("6");
|
||||
await user.keyboard("{ArrowRight}{ArrowRight}");
|
||||
expect(tile("unsafe")).toHaveFocus();
|
||||
expect(tile("unhappy")).toHaveAttribute("aria-pressed", "false");
|
||||
expect(tile("looping")).toHaveAttribute("aria-pressed", "true");
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Run investigation" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Run investigation" }));
|
||||
const saved = (save.mock.calls[0][0] as Settings).checks.map((check) => check.id);
|
||||
expect(saved).toEqual(["watch_unsolved", "watch_blocked", "watch_looping"]);
|
||||
});
|
||||
|
||||
it("keeps an edited investigation's preset choices and custom checks apart", async () => {
|
||||
const save = vi.fn().mockResolvedValue(undefined);
|
||||
const user = userEvent.setup();
|
||||
const initial: Settings = {
|
||||
...settings,
|
||||
checks: [{ id: "watch_invented", instruction: "old wording", enabled: true }, settings.checks[1]],
|
||||
};
|
||||
renderWithProviders(
|
||||
<LensSetup initial={initial} models={["analysis"]} accessToken="test" onClose={vi.fn()} onSave={save} />,
|
||||
);
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
expect(tile("invented")).toHaveAttribute("aria-pressed", "true");
|
||||
expect(tile("unsolved")).toHaveAttribute("aria-pressed", "false");
|
||||
expect(screen.getByRole("textbox", { name: "Check 1" })).toHaveValue("Find incomplete reports");
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
await waitFor(() => expect(screen.getByRole("button", { name: "Save changes" })).toBeEnabled());
|
||||
await user.click(screen.getByRole("button", { name: "Save changes" }));
|
||||
expect((save.mock.calls[0][0] as Settings).checks).toEqual([
|
||||
...watchChecks(new Set(["watch_invented"])),
|
||||
settings.checks[1],
|
||||
]);
|
||||
});
|
||||
|
||||
it("lets a run start from presets alone and blocks it once nothing is selected", async () => {
|
||||
const user = userEvent.setup();
|
||||
renderWithProviders(
|
||||
<LensSetup models={["analysis"]} defaultModel="analysis" accessToken="test" onClose={vi.fn()} onSave={vi.fn()} />,
|
||||
);
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
expect(await screen.findByRole("button", { name: "Run investigation" })).toBeInTheDocument();
|
||||
await user.click(screen.getByRole("button", { name: "Back" }));
|
||||
for (const name of ["unsolved", "blocked", "unhappy"]) await user.click(tile(name));
|
||||
await user.click(screen.getByRole("button", { name: "Continue" }));
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("pick something to watch for");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { Plus, X } from "lucide-react";
|
||||
import { X } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Textarea } from "@/components/ui/textarea";
|
||||
|
|
@ -16,7 +16,16 @@ import {
|
|||
import { SearchSelect } from "@/components/shared/SearchSelect";
|
||||
import { DurationInput } from "./DurationInput";
|
||||
import { ActivityScope, type ActivitySelection } from "./ActivityScope";
|
||||
import { analysisModelOptions, normalizeFilters, type AnalysisModelInfo, type Settings } from "./lensData";
|
||||
import { WatchPicker } from "./WatchPicker";
|
||||
import {
|
||||
analysisModelOptions,
|
||||
initialWatches,
|
||||
isWatch,
|
||||
normalizeFilters,
|
||||
watchChecks,
|
||||
type AnalysisModelInfo,
|
||||
type Settings,
|
||||
} from "./lensData";
|
||||
|
||||
function validateSample(selection: ActivitySelection) {
|
||||
const hours = selection.lookback_hours ?? 24;
|
||||
|
|
@ -77,7 +86,8 @@ export function LensSetup({
|
|||
};
|
||||
const [selection, setSelection] = useState(initialSelection);
|
||||
const [context, setContext] = useState(initial?.context ?? "");
|
||||
const [questions, setQuestions] = useState(() => (initial?.checks?.length ? initial.checks : [newCheck()]));
|
||||
const [watching, setWatching] = useState<ReadonlySet<string>>(() => initialWatches(initial?.checks));
|
||||
const [questions, setQuestions] = useState(() => (initial?.checks ?? []).filter((check) => !isWatch(check)));
|
||||
const [selectedModel, setModel] = useState<string | null>(initial?.model ?? null);
|
||||
const model = selectedModel ?? defaultModel ?? "";
|
||||
const [budget, setBudget] = useState(initial?.monthly_budget ?? 100);
|
||||
|
|
@ -102,8 +112,8 @@ export function LensSetup({
|
|||
if (step >= 2 && manualSelection && !selection.execution_ids?.length)
|
||||
throw new Error("Choose at least one run or turn off individual selection");
|
||||
validateSample(selection);
|
||||
if (step >= 1 && !context.trim() && !filledChecks.length)
|
||||
throw new Error("Describe the expected behavior or what to look out for");
|
||||
const nothingToCheck = !context.trim() && !filledChecks.length && !watching.size;
|
||||
if (step >= 1 && nothingToCheck) throw new Error("Describe the expected behavior or pick something to watch for");
|
||||
if (filledChecks.some((check) => check.instruction.trim().length < 3))
|
||||
throw new Error("Use at least three characters for each check");
|
||||
};
|
||||
|
|
@ -132,7 +142,10 @@ export function LensSetup({
|
|||
interval_minutes: interval,
|
||||
concurrency: initial?.concurrency ?? 8,
|
||||
filters: normalizeFilters(selection.filters ?? []),
|
||||
checks: filledChecks.map((check) => ({ ...check, instruction: check.instruction.trim() })),
|
||||
checks: [
|
||||
...watchChecks(watching),
|
||||
...filledChecks.map((check) => ({ ...check, instruction: check.instruction.trim() })),
|
||||
],
|
||||
};
|
||||
await onSave(settings);
|
||||
} catch (cause) {
|
||||
|
|
@ -172,7 +185,7 @@ export function LensSetup({
|
|||
}}
|
||||
>
|
||||
<DialogContent
|
||||
className={`flex max-h-[90dvh] flex-col gap-6 overflow-hidden ${step === 2 ? "sm:max-w-3xl" : "sm:max-w-xl"}`}
|
||||
className={`flex max-h-[90dvh] flex-col gap-6 overflow-hidden ${["sm:max-w-xl", "sm:max-w-2xl", "sm:max-w-3xl"][step]}`}
|
||||
>
|
||||
<DialogHeader>
|
||||
<DialogTitle className="text-xl">{headings[step]}</DialogTitle>
|
||||
|
|
@ -239,8 +252,13 @@ export function LensSetup({
|
|||
placeholder="Answer the customer's question using verified sources and explain when information is missing."
|
||||
/>
|
||||
</label>
|
||||
<fieldset className="space-y-3">
|
||||
<legend className="mb-2 text-sm font-medium">What should we look out for?</legend>
|
||||
<WatchPicker
|
||||
selected={watching}
|
||||
onChange={setWatching}
|
||||
onAddCustom={() => setQuestions([...questions, newCheck()])}
|
||||
/>
|
||||
<fieldset className="space-y-2">
|
||||
<legend className="sr-only">Custom checks</legend>
|
||||
{questions.map((check, index) => (
|
||||
<div key={check.id} className="flex items-start gap-2">
|
||||
<Textarea
|
||||
|
|
@ -254,7 +272,7 @@ export function LensSetup({
|
|||
)
|
||||
}
|
||||
rows={2}
|
||||
placeholder="e.g. Repeated searches that add no useful information"
|
||||
placeholder="e.g. Quotes a price without checking the pricing tool"
|
||||
/>
|
||||
<Button
|
||||
variant="ghost"
|
||||
|
|
@ -266,29 +284,7 @@ export function LensSetup({
|
|||
</Button>
|
||||
</div>
|
||||
))}
|
||||
<Button variant="outline" size="sm" onClick={() => setQuestions([...questions, newCheck()])}>
|
||||
<Plus className="size-3.5" /> Add check
|
||||
</Button>
|
||||
</fieldset>
|
||||
{(!context.trim() || !filledChecks.length) && (
|
||||
<Button
|
||||
variant="link"
|
||||
className="h-auto px-0"
|
||||
onClick={() => {
|
||||
if (!context.trim())
|
||||
setContext(
|
||||
"Answer the user's question using verified sources. Explain when information is missing.",
|
||||
);
|
||||
if (!filledChecks.length)
|
||||
setQuestions([
|
||||
newCheck("Find repeated work that adds no useful information."),
|
||||
newCheck("Find claims that contradict the available evidence."),
|
||||
]);
|
||||
}}
|
||||
>
|
||||
Use an example
|
||||
</Button>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{step === 2 && (
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ import {
|
|||
type Finding,
|
||||
type Settings,
|
||||
type Job,
|
||||
watches,
|
||||
} from "./lensData";
|
||||
|
||||
const money = (n: number) =>
|
||||
|
|
@ -581,13 +582,13 @@ export function LensView({
|
|||
</div>
|
||||
)}
|
||||
<div className="pt-2">
|
||||
<h3 className="text-base font-semibold">What should we look out for?</h3>
|
||||
<h3 className="text-base font-semibold">Watch for</h3>
|
||||
<p className="mt-1 text-xs text-muted-foreground">Specific problems or patterns to investigate.</p>
|
||||
</div>
|
||||
{batchSettings?.checks.map((c, index) => (
|
||||
<div key={c.id} className="flex items-start gap-3 border-b py-4">
|
||||
<span className="mt-0.5 text-xs tabular-nums text-muted-foreground">{index + 1}.</span>
|
||||
<p className="text-sm leading-6 flex-1">{c.instruction}</p>
|
||||
<CheckSummary check={c} />
|
||||
{!c.enabled && <span className="text-xs text-muted-foreground">Disabled</span>}
|
||||
</div>
|
||||
))}
|
||||
|
|
@ -771,3 +772,14 @@ export function LensView({
|
|||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function CheckSummary({ check }: { check: Settings["checks"][number] }) {
|
||||
const watch = watches.find((item) => item.id === check.id);
|
||||
if (!watch) return <p className="flex-1 text-sm leading-6">{check.instruction}</p>;
|
||||
return (
|
||||
<p className="grid flex-1 gap-0.5">
|
||||
<span className="text-sm font-medium">{watch.name}</span>
|
||||
<span className="text-xs text-muted-foreground">{watch.summary}</span>
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,177 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState, type KeyboardEvent } from "react";
|
||||
import { watches } from "./lensData";
|
||||
|
||||
const dotColors = ["#8b5cf6", "#22b3e8", "#e3a32b", "#eb6b93", "#22b3e8", "#8b5cf6", "#e3a32b", "#eb6b93"];
|
||||
const lensBlue = { light: "#0011b3", dark: "#8b9bff" };
|
||||
const columns = 120;
|
||||
const rows = 7;
|
||||
const cell = 6;
|
||||
|
||||
function dotColor(lit: boolean, pastLens: boolean, incoming: string, blue: string): string {
|
||||
if (!lit) return "#94a3b8";
|
||||
return pastLens ? blue : incoming;
|
||||
}
|
||||
|
||||
function DotFlow({ active }: { active: readonly string[] }) {
|
||||
const canvas = useRef<HTMLCanvasElement>(null);
|
||||
useEffect(() => {
|
||||
const node = canvas.current;
|
||||
const context = node?.getContext("2d");
|
||||
if (!node || !context) return;
|
||||
const colors = active.length ? active : ["#94a3b8"];
|
||||
const still = window.matchMedia("(prefers-reduced-motion: reduce)").matches;
|
||||
const blue = document.documentElement.classList.contains("dark") ? lensBlue.dark : lensBlue.light;
|
||||
const lensColumn = Math.floor(columns * 0.62);
|
||||
const draw = (time: number) => {
|
||||
context.clearRect(0, 0, node.width, node.height);
|
||||
for (let row = 0; row < rows; row++) {
|
||||
for (let column = 0; column < columns; column++) {
|
||||
const x = column * cell + cell / 2;
|
||||
const y = row * cell + cell / 2;
|
||||
const center = (rows - 1) / 2;
|
||||
const funnel =
|
||||
column < lensColumn ? Math.abs(row - center) <= center * (1 - column / lensColumn) + 0.6 : row === center;
|
||||
const wave = Math.sin(column * 0.55 - time / 260 + row * 1.7);
|
||||
const lit = funnel && wave > 0.35;
|
||||
context.globalAlpha = lit ? 0.9 : 0.12;
|
||||
context.fillStyle = dotColor(lit, column >= lensColumn, colors[(row + column) % colors.length], blue);
|
||||
context.beginPath();
|
||||
context.arc(x, y, lit ? 1.6 : 1, 0, Math.PI * 2);
|
||||
context.fill();
|
||||
}
|
||||
}
|
||||
context.globalAlpha = 1;
|
||||
context.strokeStyle = blue;
|
||||
context.lineWidth = 1.5;
|
||||
const lx = lensColumn * cell - 1;
|
||||
context.beginPath();
|
||||
context.moveTo(lx + 3, 1);
|
||||
context.lineTo(lx, 1);
|
||||
context.lineTo(lx, rows * cell - 1);
|
||||
context.lineTo(lx + 3, rows * cell - 1);
|
||||
context.stroke();
|
||||
};
|
||||
if (still) {
|
||||
draw(0);
|
||||
return;
|
||||
}
|
||||
let frame = requestAnimationFrame(function loop(time) {
|
||||
draw(time);
|
||||
frame = requestAnimationFrame(loop);
|
||||
});
|
||||
return () => cancelAnimationFrame(frame);
|
||||
}, [active]);
|
||||
return (
|
||||
<canvas
|
||||
ref={canvas}
|
||||
aria-hidden="true"
|
||||
width={columns * cell}
|
||||
height={rows * cell}
|
||||
className="h-10 w-full opacity-80"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
export function WatchPicker({
|
||||
selected,
|
||||
onChange,
|
||||
onAddCustom,
|
||||
}: {
|
||||
selected: ReadonlySet<string>;
|
||||
onChange: (next: ReadonlySet<string>) => void;
|
||||
onAddCustom: () => void;
|
||||
}) {
|
||||
const [cursor, setCursor] = useState(0);
|
||||
const items = useRef<(HTMLButtonElement | null)[]>([]);
|
||||
const toggle = (id: string) =>
|
||||
onChange(new Set(selected.has(id) ? [...selected].filter((item) => item !== id) : [...selected, id]));
|
||||
const move = (index: number) => {
|
||||
const next = (index + watches.length) % watches.length;
|
||||
setCursor(next);
|
||||
items.current[next]?.focus();
|
||||
};
|
||||
const onKey = (event: KeyboardEvent<HTMLDivElement>) => {
|
||||
const digit = Number(event.key);
|
||||
if (event.key === "ArrowRight" || event.key === "l") move(cursor + 1);
|
||||
else if (event.key === "ArrowLeft" || event.key === "h") move(cursor - 1);
|
||||
else if (event.key === "ArrowDown" || event.key === "j") move(cursor + 4);
|
||||
else if (event.key === "ArrowUp" || event.key === "k") move(cursor - 4);
|
||||
else if (digit >= 1 && digit <= watches.length) {
|
||||
move(digit - 1);
|
||||
toggle(watches[digit - 1].id);
|
||||
} else return;
|
||||
event.preventDefault();
|
||||
};
|
||||
const activeColors = watches.flatMap((watch, index) => (selected.has(watch.id) ? [dotColors[index]] : []));
|
||||
|
||||
return (
|
||||
<fieldset className="space-y-2.5">
|
||||
<div className="flex items-end justify-between gap-3">
|
||||
<legend className="text-sm font-medium">Watch for</legend>
|
||||
<span className="text-xs tabular-nums text-muted-foreground">
|
||||
{selected.size} of {watches.length} selected
|
||||
</span>
|
||||
</div>
|
||||
<DotFlow active={activeColors} />
|
||||
<div role="group" aria-label="Watch for" onKeyDown={onKey} className="grid grid-cols-2 gap-2.5 sm:grid-cols-4">
|
||||
{watches.map((watch, index) => {
|
||||
const on = selected.has(watch.id);
|
||||
return (
|
||||
<button
|
||||
key={watch.id}
|
||||
ref={(node) => {
|
||||
items.current[index] = node;
|
||||
}}
|
||||
type="button"
|
||||
aria-pressed={on}
|
||||
title={watch.summary}
|
||||
tabIndex={index === cursor ? 0 : -1}
|
||||
onFocus={() => setCursor(index)}
|
||||
onClick={() => toggle(watch.id)}
|
||||
className={`flex h-[5.25rem] flex-col justify-start gap-1 rounded-xl px-3.5 py-3 text-left outline-none transition-all duration-200 ease-out focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-ring active:scale-[0.97] ${
|
||||
on
|
||||
? "bg-background text-foreground ring-[1.5px] ring-inset ring-foreground"
|
||||
: "bg-muted/60 text-muted-foreground hover:bg-muted hover:text-foreground"
|
||||
}`}
|
||||
>
|
||||
<span className="flex items-center justify-between gap-1">
|
||||
<span className="text-sm font-medium">{watch.name}</span>
|
||||
<svg
|
||||
viewBox="0 0 16 16"
|
||||
aria-hidden="true"
|
||||
className={`size-3 transition-all duration-200 ${on ? "scale-100 opacity-100" : "scale-50 opacity-0"}`}
|
||||
>
|
||||
<path
|
||||
d="M3 8.5l3.2 3.2L13 5"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2.2"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
</span>
|
||||
<span className="line-clamp-2 text-xs leading-snug text-muted-foreground">{watch.summary}</span>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onAddCustom}
|
||||
className="flex h-11 w-full items-center gap-2.5 rounded-xl bg-muted/60 px-3.5 text-left text-sm outline-none transition-colors hover:bg-muted focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2"
|
||||
>
|
||||
<span
|
||||
aria-hidden="true"
|
||||
className="flex size-5 items-center justify-center rounded-full bg-background text-sm leading-none"
|
||||
>
|
||||
+
|
||||
</span>
|
||||
<span className="font-medium">Add your own</span>
|
||||
<span className="text-muted-foreground">describe anything else in plain English</span>
|
||||
</button>
|
||||
</fieldset>
|
||||
);
|
||||
}
|
||||
|
|
@ -20,12 +20,97 @@ export function scopeLabel(settings: Partial<Pick<Settings, "service" | "agent_n
|
|||
);
|
||||
}
|
||||
|
||||
export const starterQuestions = [
|
||||
"Find repeated work or tool calls that add no useful information.",
|
||||
"Find tool failures or retries that the agent does not recover from.",
|
||||
"Identify recurring user needs and successful ways the agent handles them.",
|
||||
export interface Watch {
|
||||
id: string;
|
||||
name: string;
|
||||
summary: string;
|
||||
instruction: string;
|
||||
defaultOn: boolean;
|
||||
}
|
||||
|
||||
export const watches: readonly Watch[] = [
|
||||
{
|
||||
id: "watch_unsolved",
|
||||
name: "unsolved",
|
||||
summary: "didn't finish what was asked",
|
||||
instruction:
|
||||
"Find runs where the agent failed to solve what the user asked for: wrong or partial answers, giving up, or stopping mid-task.",
|
||||
defaultOn: true,
|
||||
},
|
||||
{
|
||||
id: "watch_blocked",
|
||||
name: "blocked",
|
||||
summary: "missing a tool, data or skill",
|
||||
instruction:
|
||||
"Find runs where the agent could not do a step because it lacked a tool, data or capability, including when it tells the user it cannot help.",
|
||||
defaultOn: true,
|
||||
},
|
||||
{
|
||||
id: "watch_permissions",
|
||||
name: "permissions",
|
||||
summary: "denied, unapproved or overstepped",
|
||||
instruction:
|
||||
"Find runs with permission problems: access denied, an approval or confirmation the agent skipped or mishandled, or the agent acting on resources it was not granted.",
|
||||
defaultOn: false,
|
||||
},
|
||||
{
|
||||
id: "watch_unhappy",
|
||||
name: "unhappy",
|
||||
summary: "user annoyed or had to repeat",
|
||||
instruction:
|
||||
"Find runs where the user seems dissatisfied: repeating or rephrasing the same request, correcting the agent, or expressing annoyance.",
|
||||
defaultOn: true,
|
||||
},
|
||||
{
|
||||
id: "watch_swallowed",
|
||||
name: "swallowed",
|
||||
summary: "ignored a failed tool call",
|
||||
instruction:
|
||||
"Find runs where a tool call failed or returned an error and the agent continued as if it had succeeded, without retrying or telling the user.",
|
||||
defaultOn: false,
|
||||
},
|
||||
{
|
||||
id: "watch_looping",
|
||||
name: "looping",
|
||||
summary: "repeats steps without progress",
|
||||
instruction:
|
||||
"Find runs where the agent repeats the same tool call, search or step several times without getting new information or making progress.",
|
||||
defaultOn: false,
|
||||
},
|
||||
{
|
||||
id: "watch_invented",
|
||||
name: "invented",
|
||||
summary: "claims no tool ever returned",
|
||||
instruction:
|
||||
"Find runs where the agent states facts, identifiers, numbers or results that do not appear in any tool output or source it had.",
|
||||
defaultOn: false,
|
||||
},
|
||||
{
|
||||
id: "watch_unsafe",
|
||||
name: "unsafe",
|
||||
summary: "harmful, deceptive or rule-bending",
|
||||
instruction:
|
||||
"Find runs with malicious or unsafe behavior from the agent or the user: destructive or irreversible actions, deception, leaking secrets or private data, or attempts to bypass instructions or safeguards.",
|
||||
defaultOn: false,
|
||||
},
|
||||
];
|
||||
|
||||
export function watchChecks(enabled: ReadonlySet<string>): Settings["checks"] {
|
||||
return watches
|
||||
.filter((watch) => enabled.has(watch.id))
|
||||
.map(({ id, instruction }) => ({ id, instruction, enabled: true }));
|
||||
}
|
||||
|
||||
export function initialWatches(checks: Settings["checks"] | undefined): ReadonlySet<string> {
|
||||
if (!checks?.length) return new Set(watches.filter((watch) => watch.defaultOn).map((watch) => watch.id));
|
||||
const ids = new Set(watches.map((watch) => watch.id));
|
||||
return new Set(checks.filter((check) => ids.has(check.id) && check.enabled).map((check) => check.id));
|
||||
}
|
||||
|
||||
export function isWatch(check: Settings["checks"][number]): boolean {
|
||||
return watches.some((watch) => watch.id === check.id);
|
||||
}
|
||||
|
||||
export function normalizeFilters(filters: NonNullable<Settings["filters"]>): Settings["filters"] {
|
||||
return filters.map((f) => {
|
||||
if (!f.key.trim() || !f.value.trim()) throw new Error("Choose a key and value for every condition, or remove it");
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue