From 7921716f3928ebf2d33b71012576efe0d26c31fa Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 12:57:04 -0700 Subject: [PATCH] feat(lens-ui): show findings ranked by priority with frequency and highlighted evidence (#45143) * feat(lens-ui): compute how often a finding hits sampled traces per day Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): copy a finding for an agent as markdown Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add affected, unaffected and quote highlight color tokens Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add a frequency card with stacked affected traces per day Co-Authored-By: Claude Opus 5.5 * fix(lens-ui): keep the issue brief title out of the page heading outline Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): lay out a finding as summary, fix, frequency and highlighted examples Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): show findings as a dated list with percent affected beside the open finding Co-Authored-By: Claude Opus 5.5 * test(lens-ui): cover frequency and highlighted quotes on a finding Co-Authored-By: Claude Opus 5.5 * test(lens-ui): follow findings into the split list and example cards Co-Authored-By: Claude Opus 5.5 * style(lens-ui): take finding chart and quote colors from the dashboard theme Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add a shared priority dot and pill for findings Co-Authored-By: Claude Opus 5.5 * style(lens-ui): soften the frequency card and show its date range Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): rank findings under high, medium and low priority headings Co-Authored-By: Claude Opus 5.5 * style(lens-ui): show finding priority, label quotes by content and collapse extra examples Co-Authored-By: Claude Opus 5.5 * test(lens-ui): prove findings are grouped and ordered by priority Co-Authored-By: Claude Opus 5.5 * test(lens-ui): cover finding priority, quote labels and example collapsing Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- ui/litellm-dashboard/src/app/globals.css | 12 + .../lens/LensWorkspace.integration.test.tsx | 7 +- .../FindingDetails.integration.test.tsx | 76 +++- .../lens/investigations/FindingDetails.tsx | 423 +++++++++++++----- .../FindingsView.integration.test.tsx | 47 +- .../lens/investigations/FindingsView.tsx | 215 ++++----- .../lens/investigations/FrequencyCard.tsx | 88 ++++ .../InvestigationsView.integration.test.tsx | 12 +- .../lens/investigations/IssueBrief.tsx | 4 +- .../lens/investigations/PriorityMark.tsx | 40 ++ .../components/lens/model/findings.test.ts | 31 +- .../src/components/lens/model/findings.ts | 13 + .../components/lens/model/frequency.test.ts | 64 +++ .../src/components/lens/model/frequency.ts | 58 +++ .../src/components/lens/model/inbox.ts | 4 + 15 files changed, 845 insertions(+), 249 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/FrequencyCard.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/PriorityMark.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/model/frequency.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/model/frequency.ts diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css index 3bc500c4848..36c22097cdc 100644 --- a/ui/litellm-dashboard/src/app/globals.css +++ b/ui/litellm-dashboard/src/app/globals.css @@ -211,6 +211,10 @@ --trace-row-hover: oklch(0.975 0.008 215); --trace-row-selected: oklch(0.95 0.035 200); --trace-brand: oklch(0.6 0.13 195); + --finding-affected: var(--info); + --finding-unaffected: oklch(0.551 0.027 264.364 / 0.45); + --finding-quote: color-mix(in oklab, var(--warning) 16%, transparent); + --finding-ring: 0 0 0 1px oklch(0 0 0 / 0.06), 0 1px 2px -1px oklch(0 0 0 / 0.06), 0 2px 4px 0 oklch(0 0 0 / 0.04); --trace-border: oklch(0.92 0.01 230); --trace-line: oklch(0.88 0.03 205); --trace-card-border: oklch(0.93 0.01 230); @@ -288,6 +292,10 @@ --trace-row-hover: oklch(0.23 0.018 230); --trace-row-selected: oklch(0.29 0.05 210); --trace-brand: oklch(0.78 0.13 190); + --finding-affected: var(--info); + --finding-unaffected: oklch(0.707 0.022 261.325 / 0.35); + --finding-quote: color-mix(in oklab, var(--warning) 24%, transparent); + --finding-ring: 0 0 0 1px oklch(1 0 0 / 0.08); --trace-border: oklch(0.3 0.02 235); --trace-line: oklch(0.36 0.04 210); --trace-card-border: oklch(0.27 0.02 235); @@ -326,6 +334,10 @@ --color-trace-row-hover: var(--trace-row-hover); --color-trace-row-selected: var(--trace-row-selected); --color-trace-brand: var(--trace-brand); + --color-finding-affected: var(--finding-affected); + --color-finding-unaffected: var(--finding-unaffected); + --color-finding-quote: var(--finding-quote); + --shadow-finding-ring: var(--finding-ring); --color-trace-border: var(--trace-border); --color-trace-line: var(--trace-line); --color-trace-card-border: var(--trace-card-border); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index c251b36d190..262d7ff9a51 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -141,9 +141,7 @@ describe("Lens interactive demo", () => { await user.click(await screen.findByRole("row", { name: /Repeated lookups leave customers without an answer/ })); const finding = screen.getByRole("complementary", { name: "Finding details" }); expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); - const summaries = within(finding).getAllByText("support_agent", { exact: true }); - await user.click(summaries[0]); - await user.click(within(finding).getAllByRole("button", { name: /Open original step/ })[0]); + await user.click(within(finding).getAllByRole("button", { name: "View span" })[0]); expect(await screen.findByRole("complementary", { name: "Span details" })).toHaveTextContent( "I will check that for you.", ); @@ -157,7 +155,8 @@ describe("Lens interactive demo", () => { await user.click(within(finding).getByRole("button", { name: "Back to finding" })); expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); await user.click(within(finding).getByRole("button", { name: "Close finding (Esc)" })); - expect(await screen.findByRole("table", { name: "Findings" })).toBeVisible(); + expect(await screen.findByRole("grid", { name: "Findings" })).toBeVisible(); + expect(screen.queryByRole("complementary", { name: "Finding details" })).not.toBeInTheDocument(); expect(network).not.toHaveBeenCalled(); await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true")); await expectUrl(onUrlUpdate, (url) => expect(url.has("span")).toBe(false)); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx index e705ce17e81..a6c1e60cb3b 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx @@ -116,8 +116,7 @@ it("stacks a quote's original step over the finding and keeps the feedback draft const panel = screen.getByRole("complementary", { name: "Finding details" }); const reason = () => within(panel).getByRole("textbox", { name: "What should Lens remember?", hidden: true }); fireEvent.change(reason(), { target: { value: "Draft feedback" } }); - for (const summary of within(panel).getAllByText(/quote$/)) await user.click(summary); - await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[0]); + await user.click(within(panel).getAllByRole("button", { name: "View span" })[0]); expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-1 at step-a"); expect(reason()).not.toBeVisible(); expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); @@ -127,13 +126,51 @@ it("stacks a quote's original step over the finding and keeps the feedback draft expect(reason()).toBeVisible(); expect(reason()).toHaveValue("Draft feedback"); - await user.click(within(panel).getAllByRole("button", { name: "Open original step" })[1]); + await user.click(within(panel).getAllByRole("button", { name: "View span" })[1]); expect(await within(panel).findByTestId("run-view")).toHaveTextContent("trace-2 at step-b"); const url = new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? "")); expect(url.get("evidence")).toBe(traceOf("trace-2")); expect(url.get("evidence_span")).toBe("step-b"); }); +it("reports how many sampled traces the finding affected and highlights each quoted line", () => { + const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id])); + const sampled = ["a", "b", "c", "d"].map((id) => ({ + id: traceOf(id), + name: `run ${id}`, + start_time: "2026-10-01T10:00:00Z", + metadata: [], + root_seen: true, + service: "support_agent", + source: "traces" as const, + span_count: 1, + team_id: "", + trace_id: id, + trace_ref: "", + })); + const current: Finding = { + ...finding, + occurrences: [traceOf("a")], + evidence: [{ execution_id: traceOf("a"), span_id: "s", quote: "files:read is missing", role: "support" }], + }; + renderWithLens( + + + , + ); + expect(screen.getByRole("region", { name: "Frequency" })).toHaveTextContent(/25%\s*1 of 4 traces affected/); + const example = screen.getByRole("article", { name: "run a" }); + expect(within(example).getByText("files:read is missing").tagName).toBe("MARK"); + expect(screen.queryByRole("article", { name: "run b" })).not.toBeInTheDocument(); +}); + it("shows contributing investigation runs and every affected trace, including older traces without retained quotes", async () => { const traceId = btoa(JSON.stringify(["traces", "", "older-trace", ""])); const current: Finding = { @@ -142,8 +179,33 @@ it("shows contributing investigation runs and every affected trace, including ol investigation_runs: ["first-investigation-run", "second-investigation-run"], }; renderWithLens(); - expect(screen.getByText("Found across 2 investigation runs")).toBeInTheDocument(); - expect(screen.getByText(/1 affected trace/)).toBeInTheDocument(); - fireEvent.click(screen.getByText("older-trace")); - expect(screen.getByRole("button", { name: "Open original trace" })).toBeInTheDocument(); + expect(screen.getByText("1 affected trace")).toBeVisible(); + expect(screen.getByText("Found across 2 investigation runs")).toBeVisible(); + const example = screen.getByRole("article", { name: "Trace older-tr" }); + expect(within(example).getByText("No quote was retained for this trace.")).toBeVisible(); + expect(within(example).getByRole("button", { name: "View trace" })).toBeVisible(); +}); + +it("shows the finding's priority and keeps the first three examples, revealing the rest on request", async () => { + const user = userEvent.setup(); + const traceOf = (id: string) => btoa(JSON.stringify(["traces", "", id])); + const ids = ["t1", "t2", "t3", "t4", "t5"]; + const current: Finding = { + ...finding, + occurrences: ids.map(traceOf), + evidence: ids.map((id) => ({ + execution_id: traceOf(id), + span_id: id, + quote: `Input: ${id}\nOutput: done`, + role: "support" as const, + })), + }; + renderWithLens(); + const panel = screen.getByRole("complementary", { name: "Finding details" }); + expect(within(panel).getByText("High priority")).toBeVisible(); + expect(within(panel).getAllByRole("article")).toHaveLength(3); + expect(within(panel).getAllByText("Call and result")).toHaveLength(3); + await user.click(within(panel).getByRole("button", { name: "Show 2 more examples" })); + expect(within(panel).getAllByRole("article")).toHaveLength(5); + expect(within(panel).queryByRole("button", { name: /Show \d+ more/ })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx index 95157a878d1..59ad282bbe0 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx @@ -1,23 +1,32 @@ "use client"; import { useState } from "react"; -import { ArrowUpRight } from "lucide-react"; +import { ChevronRight, ClipboardCopy, X } from "lucide-react"; import { Inspector } from "@/components/shared/Inspector"; import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; +import { useNow } from "@/hooks/useNow"; +import { copyToClipboard } from "@/utils/dataUtils"; import { AddToDatasetButton } from "../datasets/AddToDatasetDialog"; -import { evidenceTarget } from "../model/findings"; -import { runTime } from "../model/format"; +import { evidenceTarget, findingMarkdown } from "../model/findings"; +import { findingFrequency } from "../model/frequency"; +import { agoLabel, runTime } from "../model/format"; import { findingAgents, findingKey, type OwnedFinding, sampledExecutions } from "../model/inbox"; import type { Finding, Sample } from "../model/types"; import { EvidenceView } from "./Evidence"; +import { FrequencyCard } from "./FrequencyCard"; import { IssueBrief } from "./IssueBrief"; +import { PriorityPill } from "./PriorityMark"; import { type EvidenceRef, useEvidenceRoute } from "../route"; export const ownedFindingKey = (owned: OwnedFinding): string => findingKey(owned.lens, owned.finding); +type Quote = Finding["evidence"][number]; + +const SECTION_LABEL = "text-xs font-medium text-muted-foreground"; + export interface FindingDetailsProps { readonly finding: Finding; readonly lensId?: string; @@ -27,6 +36,240 @@ export interface FindingDetailsProps { readonly busy: boolean; readonly onOpenEvidence: (evidence: EvidenceRef) => void; readonly onReview: (status: Finding["status"], reason: string) => void; + readonly onClose?: () => void; +} + +function TopBar({ finding, onClose }: Pick) { + const now = useNow(30000); + return ( +
+

+ + {finding.id.slice(0, 8)} + + + + {agoLabel(Date.parse(finding.last_seen), now)} + +

+
+ + {onClose && ( + + )} +
+
+ ); +} + +function Disclosure({ title, children }: { title: string; children: React.ReactNode }) { + return ( +
+ + +
{children}
+
+ ); +} + +function ProseSection({ title, children }: { title: string; children: string }) { + return ( +
+

{title}

+

+ {children} +

+
+ ); +} + +const FIELD = /^(Input|Output|Status|Error)\s*:/gm; +const FIELD_LABEL: Readonly> = { + "Input,Output": "Call and result", + Input: "Call input", + Output: "Returned output", + Status: "Span status", + Error: "Error", +}; + +function quoteLabel(quote: Quote, isTrace: boolean): string { + if (quote.role === "counterexample") return "Counterexample"; + const fields = [...new Set(Array.from(quote.quote.matchAll(FIELD), (m) => m[1]))].join(","); + return FIELD_LABEL[fields] ?? (isTrace ? "Trace step" : "Logged request"); +} + +const MARK = { + support: "rounded-sm bg-finding-quote px-0.5 text-inherit", + counterexample: "rounded-sm bg-success/20 px-0.5 text-inherit", +} as const; + +function QuoteCard({ quote, onOpen }: { quote: Quote; onOpen: () => void }) { + const isTrace = evidenceTarget(quote.execution_id)?.source === "traces"; + return ( +
+
+ {quoteLabel(quote, isTrace)} + +
+ +
+ ); +} + +function EvidenceRail({ children }: { children: React.ReactNode }) { + return ( + <> +
+
+
+
+
+ +
Evidence
+
+
+ + + ); +} + +interface ExampleGroup { + readonly id: string; + readonly run: Sample["executions"][number] | undefined; + readonly quotes: readonly Quote[]; +} + +function Example({ group, onOpenEvidence }: { group: ExampleGroup; onOpenEvidence: (e: EvidenceRef) => void }) { + const traceId = evidenceTarget(group.id)?.id; + const name = group.run?.name ?? (traceId ? `Trace ${traceId.slice(0, 8)}` : "Recorded run"); + return ( +
+
+

+ {name} +

+ + {[group.run?.service, group.run && runTime(group.run.start_time)].filter(Boolean).join(" · ")} + +
+
+ {group.quotes.length === 0 ? ( +
+

No quote was retained for this trace.

+ +
+ ) : ( + + {group.quotes.map((quote, i) => ( + onOpenEvidence({ id: quote.execution_id, span: quote.span_id })} + /> + ))} + + )} +
+
+ ); +} + +const VISIBLE_EXAMPLES = 3; + +function Examples({ + groups, + onOpenEvidence, +}: { + groups: readonly ExampleGroup[]; + onOpenEvidence: (e: EvidenceRef) => void; +}) { + const [expanded, setExpanded] = useState(false); + if (groups.length === 0) return

No examples were recorded.

; + const shown = expanded ? groups : groups.slice(0, VISIBLE_EXAMPLES); + const hidden = groups.length - shown.length; + return ( + <> + {shown.map((group) => ( + + ))} + {hidden > 0 && ( + + )} + + ); +} + +function ReviewForm({ finding, busy, onReview }: Pick) { + const [reason, setReason] = useState(finding.reason ?? ""); + return ( +
+