From 12738b0d790efa3e4f3ae959cca06ef2936fc735 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Oct 2026 17:36:49 -0700 Subject: [PATCH] feat(lens): explain why a queued investigation is waiting Works out whether no worker is connected, the worker is busy (with its running investigations and an estimated start time), or it is just being picked up. --- .../src/components/lens/model/live.test.ts | 24 +++++- .../src/components/lens/model/live.ts | 18 ++++- .../src/components/lens/model/status.test.ts | 76 +++++++++++++++++- .../src/components/lens/model/status.ts | 80 ++++++++++++++++++- 4 files changed, 193 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/lens/model/live.test.ts b/ui/litellm-dashboard/src/components/lens/model/live.test.ts index d697fdd026b..1429965e025 100644 --- a/ui/litellm-dashboard/src/components/lens/model/live.test.ts +++ b/ui/litellm-dashboard/src/components/lens/model/live.test.ts @@ -20,6 +20,7 @@ import { queueRows, rateLabel, reviewKey, + secondsToFinishReading, shownCount, startPlayback, stepDuration, @@ -295,7 +296,10 @@ describe("strip state", () => { kind: "waiting", message: "Grouping observations…", }); - expect(stripState({ ...base, status: "queued" }, MODEL).kind).toBe("waiting"); + expect(stripState({ ...base, status: "queued" }, MODEL, "No worker connected.")).toEqual({ + kind: "waiting", + message: "No worker connected.", + }); }); it("shows the job error plainly when the run failed", () => { @@ -355,6 +359,24 @@ describe("drawer focus", () => { }); }); +describe("time left reading", () => { + const started = { + created_at: "2026-10-03T16:00:00Z", + steps: [{ kind: "stage", label: "Reading executions", at: "2026-10-03T16:00:20Z" }] as Job["steps"], + coverage: { selected: 328 } as Job["coverage"], + }; + const now = Date.parse("2026-10-03T16:01:00Z"); + + it("projects the remaining traces at the rate since reading started", () => { + expect(secondsToFinishReading({ ...started, reviewed: 80 }, now)).toBe(124); + }); + + it("has no estimate before the first review or once every trace is read", () => { + expect(secondsToFinishReading({ ...started, reviewed: 0 }, now)).toBeNull(); + expect(secondsToFinishReading({ ...started, reviewed: 328 }, now)).toBeNull(); + }); +}); + describe("short verdict", () => { it("prefers the issue, otherwise says no issues or not enough evidence", () => { expect(shortVerdict(review("a", { verdicts: [pattern("p"), issue("i", "made it up")] }))).toBe("made it up"); diff --git a/ui/litellm-dashboard/src/components/lens/model/live.ts b/ui/litellm-dashboard/src/components/lens/model/live.ts index 9a7ff4ec6fb..c5fe5209783 100644 --- a/ui/litellm-dashboard/src/components/lens/model/live.ts +++ b/ui/litellm-dashboard/src/components/lens/model/live.ts @@ -68,13 +68,17 @@ export type StripState = | { kind: "reviewing" } | { kind: "done" }; -export function stripState(job: Pick, model: string): StripState { +export function stripState( + job: Pick, + model: string, + queued = "Queued, waiting for a worker to pick this up", +): StripState { if (job.status === "failed") return { kind: "failed", message: job.error || "The investigation failed" }; const stepError = job.steps.findLast((step) => step.kind === "error"); if (stepError && !job.reviews.length) return { kind: "failed", message: stepError.label }; if (job.status === "completed" || job.status === "cancelled") return { kind: "done" }; if (job.reviews.length) return { kind: "reviewing" }; - if (job.status === "queued") return { kind: "waiting", message: "Queued, waiting for a worker to pick this up" }; + if (job.status === "queued") return { kind: "waiting", message: queued }; const { selected } = job.coverage; const using = model ? ` with ${model}` : ""; if (job.stage === "Reading executions" && selected) { @@ -182,6 +186,16 @@ export function readingStart(job: Pick): string { return job.steps.find((step) => step.kind === "stage" && step.label === "Reading executions")?.at ?? job.created_at; } +export function secondsToFinishReading( + job: Pick, + now: number, +): number | null { + const remaining = job.coverage.selected - job.reviewed; + const elapsed = (now - Date.parse(readingStart(job))) / 1000; + if (remaining <= 0 || job.reviewed <= 0 || elapsed <= 0) return null; + return Math.ceil(remaining / (job.reviewed / elapsed)); +} + export function liveStats(job: Job, now: number): LiveStats { const models = job.steps.filter((step) => step.kind === "model"); const end = job.finished_at ? Date.parse(job.finished_at) : now; diff --git a/ui/litellm-dashboard/src/components/lens/model/status.test.ts b/ui/litellm-dashboard/src/components/lens/model/status.test.ts index 2658d2bc40b..a8a52f9644c 100644 --- a/ui/litellm-dashboard/src/components/lens/model/status.test.ts +++ b/ui/litellm-dashboard/src/components/lens/model/status.test.ts @@ -1,4 +1,4 @@ -import { nextCheckStatus, workerConnected } from "./status"; +import { nextCheckStatus, queueReason, queueReasonText, workerConnected, workerTaskText } from "./status"; import { describe, expect, it } from "vitest"; import type { Job, Lens, LensList } from "./types"; @@ -73,6 +73,80 @@ describe("Worker readiness", () => { }); }); +describe("Why a run is queued", () => { + const now = Date.parse("2026-10-01T12:00:00Z"); + const worker: LensList["workers"][number] = { + id: "w1", + name: "Worker", + revoked: false, + analysis_key_id: "key", + last_seen: "2026-10-01T11:59:59Z", + scope: { all_teams: true, api_key_hash: "", team_id: "" }, + }; + const queued = { ...job, id: "mine", status: "queued" as const, worker_id: null, created_at: "2026-10-01T11:59:48Z" }; + const running = (lensId: string, workerId: string, reviewed: number, status: Job["status"] = "running") => ({ + id: lensId, + settings: { ...job.settings, name: lensId }, + jobs: [ + { + ...job, + id: `job-${lensId}`, + status, + worker_id: workerId, + reviewed, + created_at: "2026-10-01T11:58:20Z", + coverage: { ...coverage, selected: 328 }, + }, + ], + }); + const three = [running("swarm", "w1", 200), running("billing", "w1", 300), running("research", "w1", 100)]; + + it("lists what a busy worker is running and when this run should start", () => { + const reason = queueReason(queued, three, [worker], now); + expect(reason.kind).toBe("busy"); + if (reason.kind !== "busy") return; + expect(reason.tasks.map(workerTaskText)).toEqual([ + "swarm · Reading executions · 200 of 328 traces", + "billing · Reading executions · 300 of 328 traces", + "research · Reading executions · 100 of 328 traces", + ]); + expect(reason.tasks[0].lensId).toBe("swarm"); + expect(reason.startsIn).toBe(10); + expect(queueReasonText(reason)).toBe("Worker is busy with 3 investigations · starts in ~10s"); + }); + + it("says no worker is connected when every heartbeat is stale or revoked", () => { + const stale = { ...worker, last_seen: "2026-10-01T11:50:00Z" }; + expect(queueReason(queued, three, [stale, { ...worker, revoked: true }], now)).toEqual({ kind: "no_worker" }); + expect(queueReasonText({ kind: "no_worker" })).toBe("No worker connected. Start one from Connect worker."); + }); + + it("is picking up when nothing else is running, counting seconds waited", () => { + const done = [running("swarm", "w1", 328, "completed")]; + expect(queueReason(queued, done, [worker], now)).toEqual({ kind: "starting", seconds: 12, tasks: [] }); + expect(queueReasonText({ kind: "starting", seconds: 12, tasks: [] })).toBe("Picking up… 12s"); + }); + + it("gives the worker a moment to pick up before calling it busy", () => { + const fresh = { ...queued, created_at: "2026-10-01T11:59:57Z" }; + expect(queueReason(fresh, three, [worker], now).kind).toBe("starting"); + }); + + it("only counts work on the worker this run is assigned to, and never itself", () => { + const free = { ...worker, id: "w2" }; + const elsewhere = [running("swarm", "w2", 200)]; + expect(queueReason({ ...queued, worker_id: "w1" }, elsewhere, [worker, free], now).kind).toBe("starting"); + const self = { id: "mine", settings: job.settings, jobs: [{ ...queued, status: "running" as const, worker_id: "w1" }] }; + expect(queueReason(queued, [self], [worker], now).kind).toBe("starting"); + }); + + it("has no start estimate before any busy run has a rate", () => { + const reason = queueReason(queued, [running("swarm", "w1", 0)], [worker], now); + expect(reason).toMatchObject({ kind: "busy", startsIn: null }); + expect(queueReasonText(reason)).toBe("Worker is busy with 1 investigation"); + }); +}); + const lens: Lens = { version: 0, spent: 0, diff --git a/ui/litellm-dashboard/src/components/lens/model/status.ts b/ui/litellm-dashboard/src/components/lens/model/status.ts index daed4c9060b..bd7b723be17 100644 --- a/ui/litellm-dashboard/src/components/lens/model/status.ts +++ b/ui/litellm-dashboard/src/components/lens/model/status.ts @@ -1,10 +1,88 @@ import { formatActivityTimestamp } from "@/utils/activityTimestamp"; -import type { Lens, LensList } from "./types"; +import { secondsToFinishReading } from "./live"; +import type { Job, Lens, LensList } from "./types"; export function workerConnected(worker: LensList["workers"][number], now = Date.now()): boolean { return !worker.revoked && !!worker.analysis_key_id && now - Date.parse(worker.last_seen) < 120000; } +const PICKUP_GRACE_SECONDS = 6; + +export interface WorkerTask { + lensId: string; + name: string; + stage: string; + reviewed: number; + selected: number; + secondsLeft: number | null; +} + +export type QueueReason = + | { kind: "busy"; tasks: readonly WorkerTask[]; startsIn: number | null } + | { kind: "no_worker" } + | { kind: "starting"; seconds: number; tasks: readonly WorkerTask[] }; + +type QueuedJob = Pick; + +function workerTasks(job: QueuedJob, lenses: readonly Pick[], workerIds: ReadonlySet, now: number) { + return lenses.flatMap((lens) => + lens.jobs + .filter((other) => other.status === "running" && other.id !== job.id && workerIds.has(other.worker_id ?? "")) + .map((other) => ({ + workerId: other.worker_id ?? "", + task: { + lensId: lens.id, + name: lens.settings.name, + stage: other.stage, + reviewed: other.reviewed, + selected: other.coverage.selected, + secondsLeft: secondsToFinishReading(other, now), + }, + })), + ); +} + +export function queueReason( + job: QueuedJob, + lenses: readonly Pick[], + workers: readonly LensList["workers"][number][], + now: number, +): QueueReason { + const connected = workers.filter((worker) => workerConnected(worker, now)).map((worker) => worker.id); + if (!connected.length) return { kind: "no_worker" }; + const eligible = new Set(job.worker_id ? [job.worker_id] : connected); + const running = workerTasks(job, lenses, eligible, now); + const tasks = running.map((entry) => entry.task); + const seconds = Math.max(0, Math.floor((now - Date.parse(job.created_at)) / 1000)); + if (!tasks.length || seconds < PICKUP_GRACE_SECONDS) return { kind: "starting", seconds, tasks }; + const estimates = tasks.flatMap((task) => (task.secondsLeft === null ? [] : [task.secondsLeft])); + return { kind: "busy", tasks, startsIn: estimates.length ? Math.min(...estimates) : null }; +} + +function waitLabel(seconds: number): string { + if (seconds < 60) return `~${Math.max(5, Math.ceil(seconds / 5) * 5)}s`; + return `~${Math.ceil(seconds / 60)}m`; +} + +export function queueReasonText(reason: QueueReason): string { + switch (reason.kind) { + case "busy": { + const count = reason.tasks.length; + const when = reason.startsIn === null ? "" : ` · starts in ${waitLabel(reason.startsIn)}`; + return `Worker is busy with ${count} ${count === 1 ? "investigation" : "investigations"}${when}`; + } + case "no_worker": + return "No worker connected. Start one from Connect worker."; + case "starting": + return `Picking up… ${reason.seconds}s`; + } +} + +export function workerTaskText(task: WorkerTask): string { + const progress = task.selected ? ` · ${task.reviewed} of ${task.selected} traces` : ""; + return `${task.name} · ${task.stage || "starting"}${progress}`; +} + export function lensStatus(lens: Lens, connected: boolean): string { const active = lens.jobs?.find((job) => ["queued", "running"].includes(job.status ?? "")); if (active) return connected ? active.stage ?? "Queued" : "Waiting for analyzer";