mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
feat(lens): explain why a queued investigation is waiting
Works out whether no worker is connected, the worker is busy (with its running investigations and an estimated start time), or it is just being picked up.
This commit is contained in:
parent
15e570e4c7
commit
12738b0d79
4 changed files with 193 additions and 5 deletions
|
|
@ -20,6 +20,7 @@ import {
|
|||
queueRows,
|
||||
rateLabel,
|
||||
reviewKey,
|
||||
secondsToFinishReading,
|
||||
shownCount,
|
||||
startPlayback,
|
||||
stepDuration,
|
||||
|
|
@ -295,7 +296,10 @@ describe("strip state", () => {
|
|||
kind: "waiting",
|
||||
message: "Grouping observations…",
|
||||
});
|
||||
expect(stripState({ ...base, status: "queued" }, MODEL).kind).toBe("waiting");
|
||||
expect(stripState({ ...base, status: "queued" }, MODEL, "No worker connected.")).toEqual({
|
||||
kind: "waiting",
|
||||
message: "No worker connected.",
|
||||
});
|
||||
});
|
||||
|
||||
it("shows the job error plainly when the run failed", () => {
|
||||
|
|
@ -355,6 +359,24 @@ describe("drawer focus", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("time left reading", () => {
|
||||
const started = {
|
||||
created_at: "2026-10-03T16:00:00Z",
|
||||
steps: [{ kind: "stage", label: "Reading executions", at: "2026-10-03T16:00:20Z" }] as Job["steps"],
|
||||
coverage: { selected: 328 } as Job["coverage"],
|
||||
};
|
||||
const now = Date.parse("2026-10-03T16:01:00Z");
|
||||
|
||||
it("projects the remaining traces at the rate since reading started", () => {
|
||||
expect(secondsToFinishReading({ ...started, reviewed: 80 }, now)).toBe(124);
|
||||
});
|
||||
|
||||
it("has no estimate before the first review or once every trace is read", () => {
|
||||
expect(secondsToFinishReading({ ...started, reviewed: 0 }, now)).toBeNull();
|
||||
expect(secondsToFinishReading({ ...started, reviewed: 328 }, now)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("short verdict", () => {
|
||||
it("prefers the issue, otherwise says no issues or not enough evidence", () => {
|
||||
expect(shortVerdict(review("a", { verdicts: [pattern("p"), issue("i", "made it up")] }))).toBe("made it up");
|
||||
|
|
|
|||
|
|
@ -68,13 +68,17 @@ export type StripState =
|
|||
| { kind: "reviewing" }
|
||||
| { kind: "done" };
|
||||
|
||||
export function stripState(job: Pick<Job, "status" | "error" | "stage" | "steps" | "coverage" | "reviews">, model: string): StripState {
|
||||
export function stripState(
|
||||
job: Pick<Job, "status" | "error" | "stage" | "steps" | "coverage" | "reviews">,
|
||||
model: string,
|
||||
queued = "Queued, waiting for a worker to pick this up",
|
||||
): StripState {
|
||||
if (job.status === "failed") return { kind: "failed", message: job.error || "The investigation failed" };
|
||||
const stepError = job.steps.findLast((step) => step.kind === "error");
|
||||
if (stepError && !job.reviews.length) return { kind: "failed", message: stepError.label };
|
||||
if (job.status === "completed" || job.status === "cancelled") return { kind: "done" };
|
||||
if (job.reviews.length) return { kind: "reviewing" };
|
||||
if (job.status === "queued") return { kind: "waiting", message: "Queued, waiting for a worker to pick this up" };
|
||||
if (job.status === "queued") return { kind: "waiting", message: queued };
|
||||
const { selected } = job.coverage;
|
||||
const using = model ? ` with ${model}` : "";
|
||||
if (job.stage === "Reading executions" && selected) {
|
||||
|
|
@ -182,6 +186,16 @@ export function readingStart(job: Pick<Job, "steps" | "created_at">): string {
|
|||
return job.steps.find((step) => step.kind === "stage" && step.label === "Reading executions")?.at ?? job.created_at;
|
||||
}
|
||||
|
||||
export function secondsToFinishReading(
|
||||
job: Pick<Job, "steps" | "created_at" | "reviewed" | "coverage">,
|
||||
now: number,
|
||||
): number | null {
|
||||
const remaining = job.coverage.selected - job.reviewed;
|
||||
const elapsed = (now - Date.parse(readingStart(job))) / 1000;
|
||||
if (remaining <= 0 || job.reviewed <= 0 || elapsed <= 0) return null;
|
||||
return Math.ceil(remaining / (job.reviewed / elapsed));
|
||||
}
|
||||
|
||||
export function liveStats(job: Job, now: number): LiveStats {
|
||||
const models = job.steps.filter((step) => step.kind === "model");
|
||||
const end = job.finished_at ? Date.parse(job.finished_at) : now;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { nextCheckStatus, workerConnected } from "./status";
|
||||
import { nextCheckStatus, queueReason, queueReasonText, workerConnected, workerTaskText } from "./status";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { Job, Lens, LensList } from "./types";
|
||||
|
|
@ -73,6 +73,80 @@ describe("Worker readiness", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("Why a run is queued", () => {
|
||||
const now = Date.parse("2026-10-01T12:00:00Z");
|
||||
const worker: LensList["workers"][number] = {
|
||||
id: "w1",
|
||||
name: "Worker",
|
||||
revoked: false,
|
||||
analysis_key_id: "key",
|
||||
last_seen: "2026-10-01T11:59:59Z",
|
||||
scope: { all_teams: true, api_key_hash: "", team_id: "" },
|
||||
};
|
||||
const queued = { ...job, id: "mine", status: "queued" as const, worker_id: null, created_at: "2026-10-01T11:59:48Z" };
|
||||
const running = (lensId: string, workerId: string, reviewed: number, status: Job["status"] = "running") => ({
|
||||
id: lensId,
|
||||
settings: { ...job.settings, name: lensId },
|
||||
jobs: [
|
||||
{
|
||||
...job,
|
||||
id: `job-${lensId}`,
|
||||
status,
|
||||
worker_id: workerId,
|
||||
reviewed,
|
||||
created_at: "2026-10-01T11:58:20Z",
|
||||
coverage: { ...coverage, selected: 328 },
|
||||
},
|
||||
],
|
||||
});
|
||||
const three = [running("swarm", "w1", 200), running("billing", "w1", 300), running("research", "w1", 100)];
|
||||
|
||||
it("lists what a busy worker is running and when this run should start", () => {
|
||||
const reason = queueReason(queued, three, [worker], now);
|
||||
expect(reason.kind).toBe("busy");
|
||||
if (reason.kind !== "busy") return;
|
||||
expect(reason.tasks.map(workerTaskText)).toEqual([
|
||||
"swarm · Reading executions · 200 of 328 traces",
|
||||
"billing · Reading executions · 300 of 328 traces",
|
||||
"research · Reading executions · 100 of 328 traces",
|
||||
]);
|
||||
expect(reason.tasks[0].lensId).toBe("swarm");
|
||||
expect(reason.startsIn).toBe(10);
|
||||
expect(queueReasonText(reason)).toBe("Worker is busy with 3 investigations · starts in ~10s");
|
||||
});
|
||||
|
||||
it("says no worker is connected when every heartbeat is stale or revoked", () => {
|
||||
const stale = { ...worker, last_seen: "2026-10-01T11:50:00Z" };
|
||||
expect(queueReason(queued, three, [stale, { ...worker, revoked: true }], now)).toEqual({ kind: "no_worker" });
|
||||
expect(queueReasonText({ kind: "no_worker" })).toBe("No worker connected. Start one from Connect worker.");
|
||||
});
|
||||
|
||||
it("is picking up when nothing else is running, counting seconds waited", () => {
|
||||
const done = [running("swarm", "w1", 328, "completed")];
|
||||
expect(queueReason(queued, done, [worker], now)).toEqual({ kind: "starting", seconds: 12, tasks: [] });
|
||||
expect(queueReasonText({ kind: "starting", seconds: 12, tasks: [] })).toBe("Picking up… 12s");
|
||||
});
|
||||
|
||||
it("gives the worker a moment to pick up before calling it busy", () => {
|
||||
const fresh = { ...queued, created_at: "2026-10-01T11:59:57Z" };
|
||||
expect(queueReason(fresh, three, [worker], now).kind).toBe("starting");
|
||||
});
|
||||
|
||||
it("only counts work on the worker this run is assigned to, and never itself", () => {
|
||||
const free = { ...worker, id: "w2" };
|
||||
const elsewhere = [running("swarm", "w2", 200)];
|
||||
expect(queueReason({ ...queued, worker_id: "w1" }, elsewhere, [worker, free], now).kind).toBe("starting");
|
||||
const self = { id: "mine", settings: job.settings, jobs: [{ ...queued, status: "running" as const, worker_id: "w1" }] };
|
||||
expect(queueReason(queued, [self], [worker], now).kind).toBe("starting");
|
||||
});
|
||||
|
||||
it("has no start estimate before any busy run has a rate", () => {
|
||||
const reason = queueReason(queued, [running("swarm", "w1", 0)], [worker], now);
|
||||
expect(reason).toMatchObject({ kind: "busy", startsIn: null });
|
||||
expect(queueReasonText(reason)).toBe("Worker is busy with 1 investigation");
|
||||
});
|
||||
});
|
||||
|
||||
const lens: Lens = {
|
||||
version: 0,
|
||||
spent: 0,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,88 @@
|
|||
import { formatActivityTimestamp } from "@/utils/activityTimestamp";
|
||||
import type { Lens, LensList } from "./types";
|
||||
import { secondsToFinishReading } from "./live";
|
||||
import type { Job, Lens, LensList } from "./types";
|
||||
|
||||
export function workerConnected(worker: LensList["workers"][number], now = Date.now()): boolean {
|
||||
return !worker.revoked && !!worker.analysis_key_id && now - Date.parse(worker.last_seen) < 120000;
|
||||
}
|
||||
|
||||
const PICKUP_GRACE_SECONDS = 6;
|
||||
|
||||
export interface WorkerTask {
|
||||
lensId: string;
|
||||
name: string;
|
||||
stage: string;
|
||||
reviewed: number;
|
||||
selected: number;
|
||||
secondsLeft: number | null;
|
||||
}
|
||||
|
||||
export type QueueReason =
|
||||
| { kind: "busy"; tasks: readonly WorkerTask[]; startsIn: number | null }
|
||||
| { kind: "no_worker" }
|
||||
| { kind: "starting"; seconds: number; tasks: readonly WorkerTask[] };
|
||||
|
||||
type QueuedJob = Pick<Job, "id" | "worker_id" | "created_at">;
|
||||
|
||||
function workerTasks(job: QueuedJob, lenses: readonly Pick<Lens, "id" | "jobs" | "settings">[], workerIds: ReadonlySet<string>, now: number) {
|
||||
return lenses.flatMap((lens) =>
|
||||
lens.jobs
|
||||
.filter((other) => other.status === "running" && other.id !== job.id && workerIds.has(other.worker_id ?? ""))
|
||||
.map((other) => ({
|
||||
workerId: other.worker_id ?? "",
|
||||
task: {
|
||||
lensId: lens.id,
|
||||
name: lens.settings.name,
|
||||
stage: other.stage,
|
||||
reviewed: other.reviewed,
|
||||
selected: other.coverage.selected,
|
||||
secondsLeft: secondsToFinishReading(other, now),
|
||||
},
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export function queueReason(
|
||||
job: QueuedJob,
|
||||
lenses: readonly Pick<Lens, "id" | "jobs" | "settings">[],
|
||||
workers: readonly LensList["workers"][number][],
|
||||
now: number,
|
||||
): QueueReason {
|
||||
const connected = workers.filter((worker) => workerConnected(worker, now)).map((worker) => worker.id);
|
||||
if (!connected.length) return { kind: "no_worker" };
|
||||
const eligible = new Set(job.worker_id ? [job.worker_id] : connected);
|
||||
const running = workerTasks(job, lenses, eligible, now);
|
||||
const tasks = running.map((entry) => entry.task);
|
||||
const seconds = Math.max(0, Math.floor((now - Date.parse(job.created_at)) / 1000));
|
||||
if (!tasks.length || seconds < PICKUP_GRACE_SECONDS) return { kind: "starting", seconds, tasks };
|
||||
const estimates = tasks.flatMap((task) => (task.secondsLeft === null ? [] : [task.secondsLeft]));
|
||||
return { kind: "busy", tasks, startsIn: estimates.length ? Math.min(...estimates) : null };
|
||||
}
|
||||
|
||||
function waitLabel(seconds: number): string {
|
||||
if (seconds < 60) return `~${Math.max(5, Math.ceil(seconds / 5) * 5)}s`;
|
||||
return `~${Math.ceil(seconds / 60)}m`;
|
||||
}
|
||||
|
||||
export function queueReasonText(reason: QueueReason): string {
|
||||
switch (reason.kind) {
|
||||
case "busy": {
|
||||
const count = reason.tasks.length;
|
||||
const when = reason.startsIn === null ? "" : ` · starts in ${waitLabel(reason.startsIn)}`;
|
||||
return `Worker is busy with ${count} ${count === 1 ? "investigation" : "investigations"}${when}`;
|
||||
}
|
||||
case "no_worker":
|
||||
return "No worker connected. Start one from Connect worker.";
|
||||
case "starting":
|
||||
return `Picking up… ${reason.seconds}s`;
|
||||
}
|
||||
}
|
||||
|
||||
export function workerTaskText(task: WorkerTask): string {
|
||||
const progress = task.selected ? ` · ${task.reviewed} of ${task.selected} traces` : "";
|
||||
return `${task.name} · ${task.stage || "starting"}${progress}`;
|
||||
}
|
||||
|
||||
export function lensStatus(lens: Lens, connected: boolean): string {
|
||||
const active = lens.jobs?.find((job) => ["queued", "running"].includes(job.status ?? ""));
|
||||
if (active) return connected ? active.stage ?? "Queued" : "Waiting for analyzer";
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue