From d16b0d71329aa0cc62fba6a22460e44f920a5b02 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Oct 2026 17:00:31 -0700 Subject: [PATCH] feat(lens): derive strip status, honest issue counts and drawer focus from a job --- .../src/components/lens/model/live.test.ts | 89 +++++++++++++++++++ .../src/components/lens/model/live.ts | 49 ++++++++++ 2 files changed, 138 insertions(+) diff --git a/ui/litellm-dashboard/src/components/lens/model/live.test.ts b/ui/litellm-dashboard/src/components/lens/model/live.test.ts index e020b0f1ca0..ed9895031f4 100644 --- a/ui/litellm-dashboard/src/components/lens/model/live.test.ts +++ b/ui/litellm-dashboard/src/components/lens/model/live.test.ts @@ -2,6 +2,10 @@ import { describe, expect, it } from "vitest"; import { analysisModel, conclusions, + focusedReview, + issueCount, + shortVerdict, + stripState, tickerLine, liveJob, liveStats, @@ -229,6 +233,91 @@ describe("which job the live run shows", () => { }); }); +describe("strip state", () => { + const base = { + status: "running" as Job["status"], + error: "", + stage: "Reading executions", + steps: [] as Job["steps"], + coverage: { selected: 328 } as Job["coverage"], + reviews: [] as Review[], + }; + const MODEL = "cerebras/gpt-oss-120b"; + + it("says what is happening before the first review instead of a silent spinner", () => { + expect(stripState(base, MODEL)).toEqual({ kind: "waiting", message: "Reading 328 traces with cerebras/gpt-oss-120b…" }); + expect(stripState({ ...base, stage: "Grouping observations" }, "")).toEqual({ + kind: "waiting", + message: "Grouping observations…", + }); + expect(stripState({ ...base, status: "queued" }, MODEL).kind).toBe("waiting"); + }); + + it("shows the job error plainly when the run failed", () => { + expect(stripState({ ...base, status: "failed", error: "Anthropic: credit balance too low" }, MODEL)).toEqual({ + kind: "failed", + message: "Anthropic: credit balance too low", + }); + expect(stripState({ ...base, status: "failed" }, MODEL)).toEqual({ kind: "failed", message: "The investigation failed" }); + }); + + it("surfaces a model error step while still running with nothing reviewed", () => { + const steps = [{ kind: "error", label: "Model call failed: 400 credit balance too low" }] as Job["steps"]; + expect(stripState({ ...base, steps }, MODEL)).toEqual({ + kind: "failed", + message: "Model call failed: 400 credit balance too low", + }); + expect(stripState({ ...base, steps, reviews: [review("a")] }, MODEL).kind).toBe("reviewing"); + }); + + it("is done for finished runs and reviewing once reviews arrive", () => { + expect(stripState({ ...base, status: "completed" }, MODEL).kind).toBe("done"); + expect(stripState({ ...base, reviews: [review("a")] }, MODEL).kind).toBe("reviewing"); + }); +}); + +describe("issue count", () => { + const issueReview = review("x", { verdicts: [issue("i")] }); + + it("uses the job's issue findings once the run completes", () => { + const findings = [{ kind: "issue" }, { kind: "pattern" }, { kind: "issue" }] as Job["findings"]; + expect(issueCount({ status: "completed", findings, reviews: [issueReview], reviewed: 328 })).toEqual({ + count: 2, + scope: "findings", + }); + }); + + it("labels counts honestly when only the last reviews are kept", () => { + const reviews = [issueReview, review("y"), issueReview]; + expect(issueCount({ status: "running", findings: null, reviews, reviewed: 3 })).toEqual({ count: 2, scope: "" }); + expect(issueCount({ status: "running", findings: null, reviews, reviewed: 120 })).toEqual({ + count: 2, + scope: "in last 3 reviewed", + }); + }); +}); + +describe("drawer focus", () => { + const reviews = [review("a"), review("b")]; + + it("follows the live review until one is picked", () => { + expect(focusedReview(reviews, null, reviews[1])).toEqual({ review: reviews[1], following: true }); + expect(focusedReview(reviews, reviewKey(reviews[0]), reviews[1])).toEqual({ review: reviews[0], following: false }); + }); + + it("goes back to live when the picked review was dropped by the cap", () => { + expect(focusedReview(reviews, "gone@x", reviews[1])).toEqual({ review: reviews[1], following: true }); + }); +}); + +describe("short verdict", () => { + it("prefers the issue, otherwise says no issues or not enough evidence", () => { + expect(shortVerdict(review("a", { verdicts: [pattern("p"), issue("i", "made it up")] }))).toBe("made it up"); + expect(shortVerdict(review("a", { verdicts: [pattern("p")] }))).toBe("no issues"); + expect(shortVerdict(review("a", { cannot_assess: true }))).toBe("not enough evidence"); + }); +}); + describe("live stats", () => { const job = { reviewed: 30, diff --git a/ui/litellm-dashboard/src/components/lens/model/live.ts b/ui/litellm-dashboard/src/components/lens/model/live.ts index f5651c7c922..2780523857d 100644 --- a/ui/litellm-dashboard/src/components/lens/model/live.ts +++ b/ui/litellm-dashboard/src/components/lens/model/live.ts @@ -55,6 +55,55 @@ export function tickerLine(review: Pick): return `reading ${review.agent || review.name} · ${review.trace_id.slice(0, 8)}`; } +export function shortVerdict(review: Pick): string { + const issue = review.verdicts.find((v) => v.kind === "issue"); + if (issue) return issue.summary; + return review.cannot_assess ? "not enough evidence" : "no issues"; +} + +export type StripState = + | { kind: "failed"; message: string } + | { kind: "waiting"; message: string } + | { kind: "reviewing" } + | { kind: "done" }; + +export function stripState(job: Pick, model: string): StripState { + if (job.status === "failed") return { kind: "failed", message: job.error || "The investigation failed" }; + const stepError = job.steps.findLast((step) => step.kind === "error"); + if (stepError && !job.reviews.length) return { kind: "failed", message: stepError.label }; + if (job.status === "completed" || job.status === "cancelled") return { kind: "done" }; + if (job.reviews.length) return { kind: "reviewing" }; + if (job.status === "queued") return { kind: "waiting", message: "Queued, waiting for a worker to pick this up" }; + const { selected } = job.coverage; + const using = model ? ` with ${model}` : ""; + if (job.stage === "Reading executions" && selected) { + return { kind: "waiting", message: `Reading ${selected} ${selected === 1 ? "trace" : "traces"}${using}…` }; + } + return { kind: "waiting", message: `${job.stage || "Starting"}${using}…` }; +} + +export interface IssueCount { + count: number; + scope: string; +} + +export function issueCount(job: Pick): IssueCount { + const findings = job.findings?.filter((f) => f.kind === "issue").length; + if (job.status === "completed" && findings !== undefined) return { count: findings, scope: "findings" }; + const count = job.reviews.filter((r) => outcome(r) === "issue").length; + if (job.reviewed > job.reviews.length) return { count, scope: `in last ${job.reviews.length} reviewed` }; + return { count, scope: "" }; +} + +export function focusedReview( + reviews: readonly Review[], + pinned: string | null, + live: Review | null, +): { review: Review | null; following: boolean } { + const picked = pinned ? reviews.find((r) => reviewKey(r) === pinned) : undefined; + return picked ? { review: picked, following: false } : { review: live, following: true }; +} + export function verdictLine(review: Pick): string { const issue = review.verdicts.find((v) => v.kind === "issue"); if (issue) return issue.summary;