From e8f49b18ca335a82dfc5501fef03f5abe3c4d7fd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Oct 2026 16:23:29 -0700 Subject: [PATCH] feat(lens): add live review playback model --- .../src/components/lens/model/live.test.ts | 224 ++++++++++++++++++ .../src/components/lens/model/live.ts | 205 ++++++++++++++++ 2 files changed, 429 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/lens/model/live.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/model/live.ts diff --git a/ui/litellm-dashboard/src/components/lens/model/live.test.ts b/ui/litellm-dashboard/src/components/lens/model/live.test.ts new file mode 100644 index 00000000000..08cbe8d77cd --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/model/live.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, it } from "vitest"; +import { + conclusions, + liveStats, + modelName, + outcome, + playbackPhase, + playbackReducer, + providerOf, + rateLabel, + reviewKey, + shownCount, + startPlayback, + stepDuration, + tokenLabel, + verdictLine, +} from "./live"; +import type { Job, Review } from "./types"; + +function review(id: string, overrides: Partial = {}): Review { + return { + execution_id: id, + trace_id: `trace-${id}`, + agent: "support-bot", + name: id, + spans: [], + reasoning: "", + verdicts: [], + cannot_assess: false, + model: "cerebras/gpt-oss-120b", + duration_ms: 100, + at: "2026-10-03T16:00:00Z", + ...overrides, + }; +} + +const issue = (check: string, summary = check) => ({ check_id: check, kind: "issue" as const, summary }); +const pattern = (check: string, summary = check) => ({ check_id: check, kind: "pattern" as const, summary }); + +describe("provider from model", () => { + it("takes the lowercase prefix before the first slash", () => { + expect(providerOf("Cerebras/gpt-oss-120b")).toBe("cerebras"); + expect(providerOf("openrouter/meta/llama")).toBe("openrouter"); + }); + + it("has no provider for a bare model alias", () => { + expect(providerOf("analysis")).toBe(""); + expect(providerOf("/weird")).toBe(""); + }); + + it("drops only the provider prefix from the model name", () => { + expect(modelName("openrouter/meta/llama")).toBe("meta/llama"); + expect(modelName("analysis")).toBe("analysis"); + }); +}); + +describe("review outcome", () => { + it("is an issue when any verdict is an issue", () => { + expect(outcome(review("a", { verdicts: [pattern("p"), issue("i")] }))).toBe("issue"); + }); + + it("is clear when only patterns are seen", () => { + expect(outcome(review("a", { verdicts: [pattern("p")] }))).toBe("clear"); + }); + + it("is unknown when the model cannot assess, even with an issue", () => { + expect(outcome(review("a", { cannot_assess: true, verdicts: [issue("i")] }))).toBe("unknown"); + }); + + it("leads the verdict line with the issue summary over patterns", () => { + expect(verdictLine(review("a", { verdicts: [pattern("p", "fine"), issue("i", "made it up")] }))).toBe( + "made it up", + ); + expect(verdictLine(review("a"))).toBe("No issue observed"); + expect(verdictLine(review("a", { cannot_assess: true }))).toBe("Not enough evidence to judge"); + }); +}); + +describe("conclusions", () => { + it("groups verdicts by check, counts them and ranks issues before patterns, then by count", () => { + const reviews = [ + review("a", { verdicts: [pattern("calm"), issue("invented", "first")] }), + review("b", { verdicts: [pattern("calm"), pattern("calm")] }), + review("c", { verdicts: [issue("unhappy")] }), + review("d", { verdicts: [issue("invented", "latest")] }), + ]; + const result = conclusions(reviews, [{ id: "invented", instruction: "Invents answers", enabled: true }]); + expect(result.map((c) => [c.checkId, c.count, c.issue])).toEqual([ + ["invented", 2, true], + ["unhappy", 1, true], + ["calm", 3, false], + ]); + expect(result[0].label).toBe("Invents answers"); + expect(result[0].latest).toBe("latest"); + expect(result[1].label).toBe("unhappy"); + }); + + it("is empty when nothing was flagged", () => { + expect(conclusions([review("a"), review("b")])).toEqual([]); + }); +}); + +describe("playback pacing", () => { + it("slows to a full window for one review and speeds up as the backlog grows", () => { + expect(stepDuration(1)).toBe(3200); + expect(stepDuration(4)).toBe(800); + expect(stepDuration(10)).toBeLessThan(stepDuration(4)); + expect(stepDuration(10_000)).toBe(140); + expect(stepDuration(0)).toBe(3200); + }); + + it("highlights spans one at a time, then types reasoning, then shows the verdict", () => { + const duration = 2000; + expect(playbackPhase(0, duration, 4, 100)).toEqual({ span: 0, typed: 0, verdict: false }); + expect(playbackPhase(duration * 0.3, duration, 4, 100).span).toBe(3); + const typing = playbackPhase(duration * 0.575, duration, 4, 100); + expect(typing).toEqual({ span: -1, typed: 50, verdict: false }); + expect(playbackPhase(duration * 0.85, duration, 4, 100)).toEqual({ span: -1, typed: 100, verdict: true }); + }); + + it("skips the animation entirely when the backlog forces short steps", () => { + expect(playbackPhase(0, 300, 4, 100)).toEqual({ span: -1, typed: 100, verdict: true }); + }); +}); + +describe("playback queue", () => { + const reviews = ["a", "b", "c", "d", "e"].map((id) => review(id)); + + it("opens a live job replaying the last few reviews and a finished job on its final review", () => { + const live = startPlayback(reviews, true); + expect(live.played.map((r) => r.execution_id)).toEqual(["a", "b"]); + expect(live.current).toBeNull(); + expect(live.pending.map((r) => r.execution_id)).toEqual(["c", "d", "e"]); + + const done = startPlayback(reviews, false); + expect(done.played.map((r) => r.execution_id)).toEqual(["a", "b", "c", "d"]); + expect(done.current?.execution_id).toBe("e"); + expect(done.pending).toEqual([]); + }); + + it("enqueues only reviews it has not seen, so repeated polls do not replay", () => { + const start = startPlayback(reviews.slice(0, 2), true); + const polled = playbackReducer(start, { type: "enqueue", reviews: reviews.slice(0, 4) }); + expect(polled.pending.map((r) => r.execution_id)).toEqual(["a", "b", "c", "d"]); + expect(playbackReducer(polled, { type: "enqueue", reviews: reviews.slice(0, 4) })).toBe(polled); + }); + + it("treats a re-reviewed execution at a new time as new", () => { + const start = startPlayback([reviews[0]], false); + const again = review("a", { at: "2026-10-03T17:00:00Z" }); + expect(reviewKey(again)).not.toBe(reviewKey(reviews[0])); + expect(playbackReducer(start, { type: "enqueue", reviews: [again] }).pending).toEqual([again]); + }); + + it("advances one review per step and holds until the step finishes", () => { + const start = startPlayback(reviews.slice(0, 2), true); + expect(start.pending).toHaveLength(2); + const first = playbackReducer(start, { type: "tick", now: 1000 }); + expect(first.current?.execution_id).toBe("a"); + expect(first.duration).toBe(stepDuration(2)); + expect(playbackReducer(first, { type: "tick", now: 1000 + first.duration - 1 })).toBe(first); + const second = playbackReducer(first, { type: "tick", now: 1000 + first.duration }); + expect(second.current?.execution_id).toBe("b"); + expect(second.played.map((r) => r.execution_id)).toEqual(["a"]); + expect(playbackReducer(second, { type: "tick", now: 1e9 })).toBe(second); + }); + + it("settles everything at once for reduced motion", () => { + const start = startPlayback(reviews, true); + const settled = playbackReducer(start, { type: "settle" }); + expect(settled.current?.execution_id).toBe("e"); + expect(settled.played.map((r) => r.execution_id)).toEqual(["a", "b", "c", "d"]); + expect(settled.pending).toEqual([]); + }); + + it("counts reviews beyond the capped list without counting the unplayed backlog", () => { + const start = startPlayback(reviews, true); + expect(shownCount(5, start)).toBe(2); + expect(shownCount(120, start)).toBe(117); + }); +}); + +describe("live stats", () => { + const job = { + reviewed: 30, + cost: 0.042, + created_at: "2026-10-03T16:00:00Z", + finished_at: null, + steps: [ + { at: "2026-10-03T16:00:05Z", kind: "stage", label: "Reading executions", model: "", purpose: "", cost: 0, + prompt_tokens: 0, completion_tokens: 0 }, + { at: "2026-10-03T16:00:06Z", kind: "model", label: "extract", model: "m", purpose: "extract", cost: 0.01, + prompt_tokens: 1200, completion_tokens: 300 }, + { at: "2026-10-03T16:00:07Z", kind: "model", label: "extract", model: "m", purpose: "extract", cost: 0.01, + prompt_tokens: 800, completion_tokens: 200 }, + ], + } as unknown as Job; + + it("measures rate from when reading started and sums model tokens", () => { + const stats = liveStats(job, Date.parse("2026-10-03T16:00:15Z")); + expect(stats.elapsedSeconds).toBe(10); + expect(stats.perSecond).toBe(3); + expect(stats.tokens).toBe(2500); + expect(stats.cost).toBe(0.042); + }); + + it("stops the clock when the job finishes", () => { + const finished = { ...job, finished_at: "2026-10-03T16:00:25Z" }; + expect(liveStats(finished, Date.parse("2026-10-03T18:00:00Z")).elapsedSeconds).toBe(20); + }); + + it("has no rate before anything is reviewed", () => { + expect(liveStats({ ...job, reviewed: 0 }, Date.parse("2026-10-03T16:00:15Z")).perSecond).toBeNull(); + }); + + it("formats rate and tokens for the footer", () => { + expect(rateLabel(null)).toBe("–"); + expect(rateLabel(2.25)).toBe("2.3 traces/s"); + expect(rateLabel(0.5)).toBe("30.0 traces/min"); + expect(tokenLabel(950)).toBe("950 tok"); + expect(tokenLabel(2500)).toBe("2.5k tok"); + expect(tokenLabel(3_400_000)).toBe("3.4M tok"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/model/live.ts b/ui/litellm-dashboard/src/components/lens/model/live.ts new file mode 100644 index 00000000000..a3346737d87 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/model/live.ts @@ -0,0 +1,205 @@ +import type { Job, Review, Settings } from "./types"; + +export type Outcome = "issue" | "clear" | "unknown"; + +export interface Conclusion { + checkId: string; + label: string; + latest: string; + count: number; + issue: boolean; +} + +export interface Phase { + span: number; + typed: number; + verdict: boolean; +} + +export interface LiveStats { + perSecond: number | null; + tokens: number; + cost: number; + elapsedSeconds: number; +} + +const PACE_WINDOW_MS = 3200; +const MIN_STEP_MS = 140; +const COMPACT_STEP_MS = 700; +const READ_SHARE = 0.35; +const TYPE_SHARE = 0.45; +const REPLAY_ON_OPEN = 3; + +const PROVIDER_ACCENT: Readonly> = { cerebras: "#f05a28" }; + +export function reviewKey(review: Pick): string { + return `${review.execution_id}@${review.at}`; +} + +export function providerOf(model: string): string { + const slash = model.indexOf("/"); + return slash > 0 ? model.slice(0, slash).toLowerCase() : ""; +} + +export function modelName(model: string): string { + return model.slice(model.indexOf("/") + 1); +} + +export function providerAccent(provider: string): string | undefined { + return PROVIDER_ACCENT[provider]; +} + +export function outcome(review: Pick): Outcome { + if (review.cannot_assess) return "unknown"; + return review.verdicts.some((v) => v.kind === "issue") ? "issue" : "clear"; +} + +export function verdictLine(review: Pick): string { + const issue = review.verdicts.find((v) => v.kind === "issue"); + if (issue) return issue.summary; + if (review.cannot_assess) return "Not enough evidence to judge"; + return review.verdicts[0]?.summary ?? "No issue observed"; +} + +export function unseen(reviews: readonly Review[], seen: ReadonlySet): Review[] { + return reviews.filter((review) => !seen.has(reviewKey(review))); +} + +export function stepDuration(backlog: number): number { + return Math.max(MIN_STEP_MS, Math.min(PACE_WINDOW_MS, Math.round(PACE_WINDOW_MS / Math.max(1, backlog)))); +} + +export function playbackPhase(elapsed: number, duration: number, spans: number, chars: number): Phase { + if (duration < COMPACT_STEP_MS) return { span: -1, typed: chars, verdict: true }; + const t = Math.max(0, elapsed) / duration; + const reading = t < READ_SHARE; + const span = reading && spans > 0 ? Math.min(spans - 1, Math.floor((t / READ_SHARE) * spans)) : -1; + const typing = Math.min(1, Math.max(0, (t - READ_SHARE) / TYPE_SHARE)); + return { span, typed: Math.round(typing * chars), verdict: t >= READ_SHARE + TYPE_SHARE }; +} + +export function conclusions(reviews: readonly Review[], checks: Settings["checks"] = []): Conclusion[] { + const instructions = new Map(checks.map((check) => [check.id, check.instruction])); + const verdicts = reviews.flatMap((review) => review.verdicts); + const grouped = verdicts.reduce( + (groups, verdict) => { + const prior = groups.get(verdict.check_id); + return new Map(groups).set(verdict.check_id, { + checkId: verdict.check_id, + label: instructions.get(verdict.check_id) ?? verdict.summary, + latest: verdict.summary, + count: (prior?.count ?? 0) + 1, + issue: (prior?.issue ?? false) || verdict.kind === "issue", + }); + }, + new Map(), + ); + return [...grouped.values()].sort((a, b) => Number(b.issue) - Number(a.issue) || b.count - a.count); +} + +export function readingStart(job: Pick): string { + return job.steps.find((step) => step.kind === "stage" && step.label === "Reading executions")?.at ?? job.created_at; +} + +export function liveStats(job: Job, now: number): LiveStats { + const models = job.steps.filter((step) => step.kind === "model"); + const end = job.finished_at ? Date.parse(job.finished_at) : now; + const elapsedSeconds = Math.max(0, Math.floor((end - Date.parse(readingStart(job))) / 1000)); + return { + perSecond: elapsedSeconds > 0 && job.reviewed > 0 ? job.reviewed / elapsedSeconds : null, + tokens: models.reduce((sum, step) => sum + step.prompt_tokens + step.completion_tokens, 0), + cost: job.cost, + elapsedSeconds, + }; +} + +export function rateLabel(perSecond: number | null): string { + if (perSecond === null) return "–"; + return perSecond >= 1 ? `${perSecond.toFixed(1)} traces/s` : `${(perSecond * 60).toFixed(1)} traces/min`; +} + +export function tokenLabel(tokens: number): string { + if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M tok`; + return tokens >= 1000 ? `${(tokens / 1000).toFixed(1)}k tok` : `${tokens} tok`; +} + +export interface Playback { + played: readonly Review[]; + current: Review | null; + pending: readonly Review[]; + seen: ReadonlySet; + startedAt: number; + duration: number; +} + +export type PlaybackAction = + | { type: "enqueue"; reviews: readonly Review[] } + | { type: "tick"; now: number } + | { type: "settle" }; + +const PLAYED_LIMIT = 200; + +export function startPlayback(reviews: readonly Review[], live: boolean): Playback { + const replay = live ? Math.min(REPLAY_ON_OPEN, reviews.length) : 0; + const shown = reviews.slice(0, reviews.length - replay); + return { + played: live ? shown : shown.slice(0, -1), + current: live ? null : shown.at(-1) ?? null, + pending: reviews.slice(shown.length), + seen: new Set(reviews.map(reviewKey)), + startedAt: Number.NEGATIVE_INFINITY, + duration: 0, + }; +} + +function enqueue(state: Playback, reviews: readonly Review[]): Playback { + const fresh = unseen(reviews, state.seen); + if (!fresh.length) return state; + return { + ...state, + pending: [...state.pending, ...fresh], + seen: new Set([...state.seen, ...fresh.map(reviewKey)]), + }; +} + +function advance(state: Playback, now: number): Playback { + const [next, ...rest] = state.pending; + if (!next) return state; + if (state.current && now - state.startedAt < state.duration) return state; + return { + ...state, + played: state.current ? [...state.played, state.current].slice(-PLAYED_LIMIT) : state.played, + current: next, + pending: rest, + startedAt: now, + duration: stepDuration(state.pending.length), + }; +} + +function settle(state: Playback): Playback { + const all = [...state.played, ...(state.current ? [state.current] : []), ...state.pending]; + return { + ...state, + played: all.slice(0, -1).slice(-PLAYED_LIMIT), + current: all.at(-1) ?? null, + pending: [], + startedAt: Number.NEGATIVE_INFINITY, + duration: 0, + }; +} + +export function playbackReducer(state: Playback, action: PlaybackAction): Playback { + switch (action.type) { + case "enqueue": + return enqueue(state, action.reviews); + case "tick": + return advance(state, action.now); + case "settle": + return settle(state); + } +} + +export function shownCount(reviewed: number, playback: Pick): number { + const local = playback.played.length + (playback.current ? 1 : 0); + return Math.max(local, reviewed - playback.pending.length); +}