feat(lens): add live review playback model

ui/litellm-dashboard/src/components/lens/model/live.test.ts

@@ -0,0 +1,224 @@
+import { describe, expect, it } from "vitest";
+import {
+  conclusions,
+  liveStats,
+  modelName,
+  outcome,
+  playbackPhase,
+  playbackReducer,
+  providerOf,
+  rateLabel,
+  reviewKey,
+  shownCount,
+  startPlayback,
+  stepDuration,
+  tokenLabel,
+  verdictLine,
+} from "./live";
+import type { Job, Review } from "./types";
+
+function review(id: string, overrides: Partial<Review> = {}): Review {
+  return {
+    execution_id: id,
+    trace_id: `trace-${id}`,
+    agent: "support-bot",
+    name: id,
+    spans: [],
+    reasoning: "",
+    verdicts: [],
+    cannot_assess: false,
+    model: "cerebras/gpt-oss-120b",
+    duration_ms: 100,
+    at: "2026-10-03T16:00:00Z",
+    ...overrides,
+  };
+}
+
+const issue = (check: string, summary = check) => ({ check_id: check, kind: "issue" as const, summary });
+const pattern = (check: string, summary = check) => ({ check_id: check, kind: "pattern" as const, summary });
+
+describe("provider from model", () => {
+  it("takes the lowercase prefix before the first slash", () => {
+    expect(providerOf("Cerebras/gpt-oss-120b")).toBe("cerebras");
+    expect(providerOf("openrouter/meta/llama")).toBe("openrouter");
+  });
+
+  it("has no provider for a bare model alias", () => {
+    expect(providerOf("analysis")).toBe("");
+    expect(providerOf("/weird")).toBe("");
+  });
+
+  it("drops only the provider prefix from the model name", () => {
+    expect(modelName("openrouter/meta/llama")).toBe("meta/llama");
+    expect(modelName("analysis")).toBe("analysis");
+  });
+});
+
+describe("review outcome", () => {
+  it("is an issue when any verdict is an issue", () => {
+    expect(outcome(review("a", { verdicts: [pattern("p"), issue("i")] }))).toBe("issue");
+  });
+
+  it("is clear when only patterns are seen", () => {
+    expect(outcome(review("a", { verdicts: [pattern("p")] }))).toBe("clear");
+  });
+
+  it("is unknown when the model cannot assess, even with an issue", () => {
+    expect(outcome(review("a", { cannot_assess: true, verdicts: [issue("i")] }))).toBe("unknown");
+  });
+
+  it("leads the verdict line with the issue summary over patterns", () => {
+    expect(verdictLine(review("a", { verdicts: [pattern("p", "fine"), issue("i", "made it up")] }))).toBe(
+      "made it up",
+    );
+    expect(verdictLine(review("a"))).toBe("No issue observed");
+    expect(verdictLine(review("a", { cannot_assess: true }))).toBe("Not enough evidence to judge");
+  });
+});
+
+describe("conclusions", () => {
+  it("groups verdicts by check, counts them and ranks issues before patterns, then by count", () => {
+    const reviews = [
+      review("a", { verdicts: [pattern("calm"), issue("invented", "first")] }),
+      review("b", { verdicts: [pattern("calm"), pattern("calm")] }),
+      review("c", { verdicts: [issue("unhappy")] }),
+      review("d", { verdicts: [issue("invented", "latest")] }),
+    ];
+    const result = conclusions(reviews, [{ id: "invented", instruction: "Invents answers", enabled: true }]);
+    expect(result.map((c) => [c.checkId, c.count, c.issue])).toEqual([
+      ["invented", 2, true],
+      ["unhappy", 1, true],
+      ["calm", 3, false],
+    ]);
+    expect(result[0].label).toBe("Invents answers");
+    expect(result[0].latest).toBe("latest");
+    expect(result[1].label).toBe("unhappy");
+  });
+
+  it("is empty when nothing was flagged", () => {
+    expect(conclusions([review("a"), review("b")])).toEqual([]);
+  });
+});
+
+describe("playback pacing", () => {
+  it("slows to a full window for one review and speeds up as the backlog grows", () => {
+    expect(stepDuration(1)).toBe(3200);
+    expect(stepDuration(4)).toBe(800);
+    expect(stepDuration(10)).toBeLessThan(stepDuration(4));
+    expect(stepDuration(10_000)).toBe(140);
+    expect(stepDuration(0)).toBe(3200);
+  });
+
+  it("highlights spans one at a time, then types reasoning, then shows the verdict", () => {
+    const duration = 2000;
+    expect(playbackPhase(0, duration, 4, 100)).toEqual({ span: 0, typed: 0, verdict: false });
+    expect(playbackPhase(duration * 0.3, duration, 4, 100).span).toBe(3);
+    const typing = playbackPhase(duration * 0.575, duration, 4, 100);
+    expect(typing).toEqual({ span: -1, typed: 50, verdict: false });
+    expect(playbackPhase(duration * 0.85, duration, 4, 100)).toEqual({ span: -1, typed: 100, verdict: true });
+  });
+
+  it("skips the animation entirely when the backlog forces short steps", () => {
+    expect(playbackPhase(0, 300, 4, 100)).toEqual({ span: -1, typed: 100, verdict: true });
+  });
+});
+
+describe("playback queue", () => {
+  const reviews = ["a", "b", "c", "d", "e"].map((id) => review(id));
+
+  it("opens a live job replaying the last few reviews and a finished job on its final review", () => {
+    const live = startPlayback(reviews, true);
+    expect(live.played.map((r) => r.execution_id)).toEqual(["a", "b"]);
+    expect(live.current).toBeNull();
+    expect(live.pending.map((r) => r.execution_id)).toEqual(["c", "d", "e"]);
+
+    const done = startPlayback(reviews, false);
+    expect(done.played.map((r) => r.execution_id)).toEqual(["a", "b", "c", "d"]);
+    expect(done.current?.execution_id).toBe("e");
+    expect(done.pending).toEqual([]);
+  });
+
+  it("enqueues only reviews it has not seen, so repeated polls do not replay", () => {
+    const start = startPlayback(reviews.slice(0, 2), true);
+    const polled = playbackReducer(start, { type: "enqueue", reviews: reviews.slice(0, 4) });
+    expect(polled.pending.map((r) => r.execution_id)).toEqual(["a", "b", "c", "d"]);
+    expect(playbackReducer(polled, { type: "enqueue", reviews: reviews.slice(0, 4) })).toBe(polled);
+  });
+
+  it("treats a re-reviewed execution at a new time as new", () => {
+    const start = startPlayback([reviews[0]], false);
+    const again = review("a", { at: "2026-10-03T17:00:00Z" });
+    expect(reviewKey(again)).not.toBe(reviewKey(reviews[0]));
+    expect(playbackReducer(start, { type: "enqueue", reviews: [again] }).pending).toEqual([again]);
+  });
+
+  it("advances one review per step and holds until the step finishes", () => {
+    const start = startPlayback(reviews.slice(0, 2), true);
+    expect(start.pending).toHaveLength(2);
+    const first = playbackReducer(start, { type: "tick", now: 1000 });
+    expect(first.current?.execution_id).toBe("a");
+    expect(first.duration).toBe(stepDuration(2));
+    expect(playbackReducer(first, { type: "tick", now: 1000 + first.duration - 1 })).toBe(first);
+    const second = playbackReducer(first, { type: "tick", now: 1000 + first.duration });
+    expect(second.current?.execution_id).toBe("b");
+    expect(second.played.map((r) => r.execution_id)).toEqual(["a"]);
+    expect(playbackReducer(second, { type: "tick", now: 1e9 })).toBe(second);
+  });
+
+  it("settles everything at once for reduced motion", () => {
+    const start = startPlayback(reviews, true);
+    const settled = playbackReducer(start, { type: "settle" });
+    expect(settled.current?.execution_id).toBe("e");
+    expect(settled.played.map((r) => r.execution_id)).toEqual(["a", "b", "c", "d"]);
+    expect(settled.pending).toEqual([]);
+  });
+
+  it("counts reviews beyond the capped list without counting the unplayed backlog", () => {
+    const start = startPlayback(reviews, true);
+    expect(shownCount(5, start)).toBe(2);
+    expect(shownCount(120, start)).toBe(117);
+  });
+});
+
+describe("live stats", () => {
+  const job = {
+    reviewed: 30,
+    cost: 0.042,
+    created_at: "2026-10-03T16:00:00Z",
+    finished_at: null,
+    steps: [
+      { at: "2026-10-03T16:00:05Z", kind: "stage", label: "Reading executions", model: "", purpose: "", cost: 0,
+        prompt_tokens: 0, completion_tokens: 0 },
+      { at: "2026-10-03T16:00:06Z", kind: "model", label: "extract", model: "m", purpose: "extract", cost: 0.01,
+        prompt_tokens: 1200, completion_tokens: 300 },
+      { at: "2026-10-03T16:00:07Z", kind: "model", label: "extract", model: "m", purpose: "extract", cost: 0.01,
+        prompt_tokens: 800, completion_tokens: 200 },
+    ],
+  } as unknown as Job;
+
+  it("measures rate from when reading started and sums model tokens", () => {
+    const stats = liveStats(job, Date.parse("2026-10-03T16:00:15Z"));
+    expect(stats.elapsedSeconds).toBe(10);
+    expect(stats.perSecond).toBe(3);
+    expect(stats.tokens).toBe(2500);
+    expect(stats.cost).toBe(0.042);
+  });
+
+  it("stops the clock when the job finishes", () => {
+    const finished = { ...job, finished_at: "2026-10-03T16:00:25Z" };
+    expect(liveStats(finished, Date.parse("2026-10-03T18:00:00Z")).elapsedSeconds).toBe(20);
+  });
+
+  it("has no rate before anything is reviewed", () => {
+    expect(liveStats({ ...job, reviewed: 0 }, Date.parse("2026-10-03T16:00:15Z")).perSecond).toBeNull();
+  });
+
+  it("formats rate and tokens for the footer", () => {
+    expect(rateLabel(null)).toBe("–");
+    expect(rateLabel(2.25)).toBe("2.3 traces/s");
+    expect(rateLabel(0.5)).toBe("30.0 traces/min");
+    expect(tokenLabel(950)).toBe("950 tok");
+    expect(tokenLabel(2500)).toBe("2.5k tok");
+    expect(tokenLabel(3_400_000)).toBe("3.4M tok");
+  });
+});

ui/litellm-dashboard/src/components/lens/model/live.ts

@@ -0,0 +1,205 @@
+import type { Job, Review, Settings } from "./types";
+
+export type Outcome = "issue" | "clear" | "unknown";
+
+export interface Conclusion {
+  checkId: string;
+  label: string;
+  latest: string;
+  count: number;
+  issue: boolean;
+}
+
+export interface Phase {
+  span: number;
+  typed: number;
+  verdict: boolean;
+}
+
+export interface LiveStats {
+  perSecond: number | null;
+  tokens: number;
+  cost: number;
+  elapsedSeconds: number;
+}
+
+const PACE_WINDOW_MS = 3200;
+const MIN_STEP_MS = 140;
+const COMPACT_STEP_MS = 700;
+const READ_SHARE = 0.35;
+const TYPE_SHARE = 0.45;
+const REPLAY_ON_OPEN = 3;
+
+const PROVIDER_ACCENT: Readonly<Record<string, string>> = { cerebras: "#f05a28" };
+
+export function reviewKey(review: Pick<Review, "execution_id" | "at">): string {
+  return `${review.execution_id}@${review.at}`;
+}
+
+export function providerOf(model: string): string {
+  const slash = model.indexOf("/");
+  return slash > 0 ? model.slice(0, slash).toLowerCase() : "";
+}
+
+export function modelName(model: string): string {
+  return model.slice(model.indexOf("/") + 1);
+}
+
+export function providerAccent(provider: string): string | undefined {
+  return PROVIDER_ACCENT[provider];
+}
+
+export function outcome(review: Pick<Review, "cannot_assess" | "verdicts">): Outcome {
+  if (review.cannot_assess) return "unknown";
+  return review.verdicts.some((v) => v.kind === "issue") ? "issue" : "clear";
+}
+
+export function verdictLine(review: Pick<Review, "cannot_assess" | "verdicts">): string {
+  const issue = review.verdicts.find((v) => v.kind === "issue");
+  if (issue) return issue.summary;
+  if (review.cannot_assess) return "Not enough evidence to judge";
+  return review.verdicts[0]?.summary ?? "No issue observed";
+}
+
+export function unseen(reviews: readonly Review[], seen: ReadonlySet<string>): Review[] {
+  return reviews.filter((review) => !seen.has(reviewKey(review)));
+}
+
+export function stepDuration(backlog: number): number {
+  return Math.max(MIN_STEP_MS, Math.min(PACE_WINDOW_MS, Math.round(PACE_WINDOW_MS / Math.max(1, backlog))));
+}
+
+export function playbackPhase(elapsed: number, duration: number, spans: number, chars: number): Phase {
+  if (duration < COMPACT_STEP_MS) return { span: -1, typed: chars, verdict: true };
+  const t = Math.max(0, elapsed) / duration;
+  const reading = t < READ_SHARE;
+  const span = reading && spans > 0 ? Math.min(spans - 1, Math.floor((t / READ_SHARE) * spans)) : -1;
+  const typing = Math.min(1, Math.max(0, (t - READ_SHARE) / TYPE_SHARE));
+  return { span, typed: Math.round(typing * chars), verdict: t >= READ_SHARE + TYPE_SHARE };
+}
+
+export function conclusions(reviews: readonly Review[], checks: Settings["checks"] = []): Conclusion[] {
+  const instructions = new Map(checks.map((check) => [check.id, check.instruction]));
+  const verdicts = reviews.flatMap((review) => review.verdicts);
+  const grouped = verdicts.reduce(
+    (groups, verdict) => {
+      const prior = groups.get(verdict.check_id);
+      return new Map(groups).set(verdict.check_id, {
+        checkId: verdict.check_id,
+        label: instructions.get(verdict.check_id) ?? verdict.summary,
+        latest: verdict.summary,
+        count: (prior?.count ?? 0) + 1,
+        issue: (prior?.issue ?? false) || verdict.kind === "issue",
+      });
+    },
+    new Map<string, Conclusion>(),
+  );
+  return [...grouped.values()].sort((a, b) => Number(b.issue) - Number(a.issue) || b.count - a.count);
+}
+
+export function readingStart(job: Pick<Job, "steps" | "created_at">): string {
+  return job.steps.find((step) => step.kind === "stage" && step.label === "Reading executions")?.at ?? job.created_at;
+}
+
+export function liveStats(job: Job, now: number): LiveStats {
+  const models = job.steps.filter((step) => step.kind === "model");
+  const end = job.finished_at ? Date.parse(job.finished_at) : now;
+  const elapsedSeconds = Math.max(0, Math.floor((end - Date.parse(readingStart(job))) / 1000));
+  return {
+    perSecond: elapsedSeconds > 0 && job.reviewed > 0 ? job.reviewed / elapsedSeconds : null,
+    tokens: models.reduce((sum, step) => sum + step.prompt_tokens + step.completion_tokens, 0),
+    cost: job.cost,
+    elapsedSeconds,
+  };
+}
+
+export function rateLabel(perSecond: number | null): string {
+  if (perSecond === null) return "–";
+  return perSecond >= 1 ? `${perSecond.toFixed(1)} traces/s` : `${(perSecond * 60).toFixed(1)} traces/min`;
+}
+
+export function tokenLabel(tokens: number): string {
+  if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M tok`;
+  return tokens >= 1000 ? `${(tokens / 1000).toFixed(1)}k tok` : `${tokens} tok`;
+}
+
+export interface Playback {
+  played: readonly Review[];
+  current: Review | null;
+  pending: readonly Review[];
+  seen: ReadonlySet<string>;
+  startedAt: number;
+  duration: number;
+}
+
+export type PlaybackAction =
+  | { type: "enqueue"; reviews: readonly Review[] }
+  | { type: "tick"; now: number }
+  | { type: "settle" };
+
+const PLAYED_LIMIT = 200;
+
+export function startPlayback(reviews: readonly Review[], live: boolean): Playback {
+  const replay = live ? Math.min(REPLAY_ON_OPEN, reviews.length) : 0;
+  const shown = reviews.slice(0, reviews.length - replay);
+  return {
+    played: live ? shown : shown.slice(0, -1),
+    current: live ? null : shown.at(-1) ?? null,
+    pending: reviews.slice(shown.length),
+    seen: new Set(reviews.map(reviewKey)),
+    startedAt: Number.NEGATIVE_INFINITY,
+    duration: 0,
+  };
+}
+
+function enqueue(state: Playback, reviews: readonly Review[]): Playback {
+  const fresh = unseen(reviews, state.seen);
+  if (!fresh.length) return state;
+  return {
+    ...state,
+    pending: [...state.pending, ...fresh],
+    seen: new Set([...state.seen, ...fresh.map(reviewKey)]),
+  };
+}
+
+function advance(state: Playback, now: number): Playback {
+  const [next, ...rest] = state.pending;
+  if (!next) return state;
+  if (state.current && now - state.startedAt < state.duration) return state;
+  return {
+    ...state,
+    played: state.current ? [...state.played, state.current].slice(-PLAYED_LIMIT) : state.played,
+    current: next,
+    pending: rest,
+    startedAt: now,
+    duration: stepDuration(state.pending.length),
+  };
+}
+
+function settle(state: Playback): Playback {
+  const all = [...state.played, ...(state.current ? [state.current] : []), ...state.pending];
+  return {
+    ...state,
+    played: all.slice(0, -1).slice(-PLAYED_LIMIT),
+    current: all.at(-1) ?? null,
+    pending: [],
+    startedAt: Number.NEGATIVE_INFINITY,
+    duration: 0,
+  };
+}
+
+export function playbackReducer(state: Playback, action: PlaybackAction): Playback {
+  switch (action.type) {
+    case "enqueue":
+      return enqueue(state, action.reviews);
+    case "tick":
+      return advance(state, action.now);
+    case "settle":
+      return settle(state);
+  }
+}
+
+export function shownCount(reviewed: number, playback: Pick<Playback, "played" | "current" | "pending">): number {
+  const local = playback.played.length + (playback.current ? 1 : 0);
+  return Math.max(local, reviewed - playback.pending.length);
+}