diff --git a/ui/litellm-dashboard/src/components/lens/model/live.test.ts b/ui/litellm-dashboard/src/components/lens/model/live.test.ts index d697fdd026b..1429965e025 100644 --- a/ui/litellm-dashboard/src/components/lens/model/live.test.ts +++ b/ui/litellm-dashboard/src/components/lens/model/live.test.ts @@ -20,6 +20,7 @@ import { queueRows, rateLabel, reviewKey, + secondsToFinishReading, shownCount, startPlayback, stepDuration, @@ -295,7 +296,10 @@ describe("strip state", () => { kind: "waiting", message: "Grouping observations…", }); - expect(stripState({ ...base, status: "queued" }, MODEL).kind).toBe("waiting"); + expect(stripState({ ...base, status: "queued" }, MODEL, "No worker connected.")).toEqual({ + kind: "waiting", + message: "No worker connected.", + }); }); it("shows the job error plainly when the run failed", () => { @@ -355,6 +359,24 @@ describe("drawer focus", () => { }); }); +describe("time left reading", () => { + const started = { + created_at: "2026-10-03T16:00:00Z", + steps: [{ kind: "stage", label: "Reading executions", at: "2026-10-03T16:00:20Z" }] as Job["steps"], + coverage: { selected: 328 } as Job["coverage"], + }; + const now = Date.parse("2026-10-03T16:01:00Z"); + + it("projects the remaining traces at the rate since reading started", () => { + expect(secondsToFinishReading({ ...started, reviewed: 80 }, now)).toBe(124); + }); + + it("has no estimate before the first review or once every trace is read", () => { + expect(secondsToFinishReading({ ...started, reviewed: 0 }, now)).toBeNull(); + expect(secondsToFinishReading({ ...started, reviewed: 328 }, now)).toBeNull(); + }); +}); + describe("short verdict", () => { it("prefers the issue, otherwise says no issues or not enough evidence", () => { expect(shortVerdict(review("a", { verdicts: [pattern("p"), issue("i", "made it up")] }))).toBe("made it up"); diff --git a/ui/litellm-dashboard/src/components/lens/model/live.ts b/ui/litellm-dashboard/src/components/lens/model/live.ts index 9a7ff4ec6fb..c5fe5209783 100644 --- a/ui/litellm-dashboard/src/components/lens/model/live.ts +++ b/ui/litellm-dashboard/src/components/lens/model/live.ts @@ -68,13 +68,17 @@ export type StripState = | { kind: "reviewing" } | { kind: "done" }; -export function stripState(job: Pick, model: string): StripState { +export function stripState( + job: Pick, + model: string, + queued = "Queued, waiting for a worker to pick this up", +): StripState { if (job.status === "failed") return { kind: "failed", message: job.error || "The investigation failed" }; const stepError = job.steps.findLast((step) => step.kind === "error"); if (stepError && !job.reviews.length) return { kind: "failed", message: stepError.label }; if (job.status === "completed" || job.status === "cancelled") return { kind: "done" }; if (job.reviews.length) return { kind: "reviewing" }; - if (job.status === "queued") return { kind: "waiting", message: "Queued, waiting for a worker to pick this up" }; + if (job.status === "queued") return { kind: "waiting", message: queued }; const { selected } = job.coverage; const using = model ? ` with ${model}` : ""; if (job.stage === "Reading executions" && selected) { @@ -182,6 +186,16 @@ export function readingStart(job: Pick): string { return job.steps.find((step) => step.kind === "stage" && step.label === "Reading executions")?.at ?? job.created_at; } +export function secondsToFinishReading( + job: Pick, + now: number, +): number | null { + const remaining = job.coverage.selected - job.reviewed; + const elapsed = (now - Date.parse(readingStart(job))) / 1000; + if (remaining <= 0 || job.reviewed <= 0 || elapsed <= 0) return null; + return Math.ceil(remaining / (job.reviewed / elapsed)); +} + export function liveStats(job: Job, now: number): LiveStats { const models = job.steps.filter((step) => step.kind === "model"); const end = job.finished_at ? Date.parse(job.finished_at) : now; diff --git a/ui/litellm-dashboard/src/components/lens/model/status.test.ts b/ui/litellm-dashboard/src/components/lens/model/status.test.ts index 2658d2bc40b..a8a52f9644c 100644 --- a/ui/litellm-dashboard/src/components/lens/model/status.test.ts +++ b/ui/litellm-dashboard/src/components/lens/model/status.test.ts @@ -1,4 +1,4 @@ -import { nextCheckStatus, workerConnected } from "./status"; +import { nextCheckStatus, queueReason, queueReasonText, workerConnected, workerTaskText } from "./status"; import { describe, expect, it } from "vitest"; import type { Job, Lens, LensList } from "./types"; @@ -73,6 +73,80 @@ describe("Worker readiness", () => { }); }); +describe("Why a run is queued", () => { + const now = Date.parse("2026-10-01T12:00:00Z"); + const worker: LensList["workers"][number] = { + id: "w1", + name: "Worker", + revoked: false, + analysis_key_id: "key", + last_seen: "2026-10-01T11:59:59Z", + scope: { all_teams: true, api_key_hash: "", team_id: "" }, + }; + const queued = { ...job, id: "mine", status: "queued" as const, worker_id: null, created_at: "2026-10-01T11:59:48Z" }; + const running = (lensId: string, workerId: string, reviewed: number, status: Job["status"] = "running") => ({ + id: lensId, + settings: { ...job.settings, name: lensId }, + jobs: [ + { + ...job, + id: `job-${lensId}`, + status, + worker_id: workerId, + reviewed, + created_at: "2026-10-01T11:58:20Z", + coverage: { ...coverage, selected: 328 }, + }, + ], + }); + const three = [running("swarm", "w1", 200), running("billing", "w1", 300), running("research", "w1", 100)]; + + it("lists what a busy worker is running and when this run should start", () => { + const reason = queueReason(queued, three, [worker], now); + expect(reason.kind).toBe("busy"); + if (reason.kind !== "busy") return; + expect(reason.tasks.map(workerTaskText)).toEqual([ + "swarm · Reading executions · 200 of 328 traces", + "billing · Reading executions · 300 of 328 traces", + "research · Reading executions · 100 of 328 traces", + ]); + expect(reason.tasks[0].lensId).toBe("swarm"); + expect(reason.startsIn).toBe(10); + expect(queueReasonText(reason)).toBe("Worker is busy with 3 investigations · starts in ~10s"); + }); + + it("says no worker is connected when every heartbeat is stale or revoked", () => { + const stale = { ...worker, last_seen: "2026-10-01T11:50:00Z" }; + expect(queueReason(queued, three, [stale, { ...worker, revoked: true }], now)).toEqual({ kind: "no_worker" }); + expect(queueReasonText({ kind: "no_worker" })).toBe("No worker connected. Start one from Connect worker."); + }); + + it("is picking up when nothing else is running, counting seconds waited", () => { + const done = [running("swarm", "w1", 328, "completed")]; + expect(queueReason(queued, done, [worker], now)).toEqual({ kind: "starting", seconds: 12, tasks: [] }); + expect(queueReasonText({ kind: "starting", seconds: 12, tasks: [] })).toBe("Picking up… 12s"); + }); + + it("gives the worker a moment to pick up before calling it busy", () => { + const fresh = { ...queued, created_at: "2026-10-01T11:59:57Z" }; + expect(queueReason(fresh, three, [worker], now).kind).toBe("starting"); + }); + + it("only counts work on the worker this run is assigned to, and never itself", () => { + const free = { ...worker, id: "w2" }; + const elsewhere = [running("swarm", "w2", 200)]; + expect(queueReason({ ...queued, worker_id: "w1" }, elsewhere, [worker, free], now).kind).toBe("starting"); + const self = { id: "mine", settings: job.settings, jobs: [{ ...queued, status: "running" as const, worker_id: "w1" }] }; + expect(queueReason(queued, [self], [worker], now).kind).toBe("starting"); + }); + + it("has no start estimate before any busy run has a rate", () => { + const reason = queueReason(queued, [running("swarm", "w1", 0)], [worker], now); + expect(reason).toMatchObject({ kind: "busy", startsIn: null }); + expect(queueReasonText(reason)).toBe("Worker is busy with 1 investigation"); + }); +}); + const lens: Lens = { version: 0, spent: 0, diff --git a/ui/litellm-dashboard/src/components/lens/model/status.ts b/ui/litellm-dashboard/src/components/lens/model/status.ts index daed4c9060b..bd7b723be17 100644 --- a/ui/litellm-dashboard/src/components/lens/model/status.ts +++ b/ui/litellm-dashboard/src/components/lens/model/status.ts @@ -1,10 +1,88 @@ import { formatActivityTimestamp } from "@/utils/activityTimestamp"; -import type { Lens, LensList } from "./types"; +import { secondsToFinishReading } from "./live"; +import type { Job, Lens, LensList } from "./types"; export function workerConnected(worker: LensList["workers"][number], now = Date.now()): boolean { return !worker.revoked && !!worker.analysis_key_id && now - Date.parse(worker.last_seen) < 120000; } +const PICKUP_GRACE_SECONDS = 6; + +export interface WorkerTask { + lensId: string; + name: string; + stage: string; + reviewed: number; + selected: number; + secondsLeft: number | null; +} + +export type QueueReason = + | { kind: "busy"; tasks: readonly WorkerTask[]; startsIn: number | null } + | { kind: "no_worker" } + | { kind: "starting"; seconds: number; tasks: readonly WorkerTask[] }; + +type QueuedJob = Pick; + +function workerTasks(job: QueuedJob, lenses: readonly Pick[], workerIds: ReadonlySet, now: number) { + return lenses.flatMap((lens) => + lens.jobs + .filter((other) => other.status === "running" && other.id !== job.id && workerIds.has(other.worker_id ?? "")) + .map((other) => ({ + workerId: other.worker_id ?? "", + task: { + lensId: lens.id, + name: lens.settings.name, + stage: other.stage, + reviewed: other.reviewed, + selected: other.coverage.selected, + secondsLeft: secondsToFinishReading(other, now), + }, + })), + ); +} + +export function queueReason( + job: QueuedJob, + lenses: readonly Pick[], + workers: readonly LensList["workers"][number][], + now: number, +): QueueReason { + const connected = workers.filter((worker) => workerConnected(worker, now)).map((worker) => worker.id); + if (!connected.length) return { kind: "no_worker" }; + const eligible = new Set(job.worker_id ? [job.worker_id] : connected); + const running = workerTasks(job, lenses, eligible, now); + const tasks = running.map((entry) => entry.task); + const seconds = Math.max(0, Math.floor((now - Date.parse(job.created_at)) / 1000)); + if (!tasks.length || seconds < PICKUP_GRACE_SECONDS) return { kind: "starting", seconds, tasks }; + const estimates = tasks.flatMap((task) => (task.secondsLeft === null ? [] : [task.secondsLeft])); + return { kind: "busy", tasks, startsIn: estimates.length ? Math.min(...estimates) : null }; +} + +function waitLabel(seconds: number): string { + if (seconds < 60) return `~${Math.max(5, Math.ceil(seconds / 5) * 5)}s`; + return `~${Math.ceil(seconds / 60)}m`; +} + +export function queueReasonText(reason: QueueReason): string { + switch (reason.kind) { + case "busy": { + const count = reason.tasks.length; + const when = reason.startsIn === null ? "" : ` · starts in ${waitLabel(reason.startsIn)}`; + return `Worker is busy with ${count} ${count === 1 ? "investigation" : "investigations"}${when}`; + } + case "no_worker": + return "No worker connected. Start one from Connect worker."; + case "starting": + return `Picking up… ${reason.seconds}s`; + } +} + +export function workerTaskText(task: WorkerTask): string { + const progress = task.selected ? ` · ${task.reviewed} of ${task.selected} traces` : ""; + return `${task.name} · ${task.stage || "starting"}${progress}`; +} + export function lensStatus(lens: Lens, connected: boolean): string { const active = lens.jobs?.find((job) => ["queued", "running"].includes(job.status ?? "")); if (active) return connected ? active.stage ?? "Queued" : "Waiting for analyzer";