diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationProgress.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationProgress.tsx index 5fc5485df96..08e5abdb01d 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationProgress.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationProgress.tsx @@ -1,8 +1,5 @@ "use client"; -import { useNow } from "@/hooks/useNow"; - -import { Button } from "@/components/ui/button"; import { analysisElapsed, analysisFraction, @@ -10,131 +7,68 @@ import { analysisProgress, analysisStages, remainingLabel, - stageDurations, } from "../model/progress"; import { useProgressSamples } from "./useProgressSamples"; -import { durationText } from "../model/format"; import { type Job } from "../model/types"; -import { cn } from "@/lib/cva.config"; import { QueueReasonText } from "./QueueReasonText"; import { useQueueReason, type QueueContext } from "./useQueueReason"; -const steps = ["review runs", "find patterns", "check evidence"]; -const markers = { done: "✓", active: "▸", todo: "·" }; -const blocks = 32; -const QUEUED_TITLE = { busy: "Waiting in line", no_worker: "Waiting for a worker", starting: "Picking up" } as const; - -function stageState(index: number, current: number): keyof typeof markers { - if (index < current) return "done"; - return index === current ? "active" : "todo"; -} - -export function InvestigationProgress({ - job, - queue, - onCancel, -}: { - job: Job; - queue?: QueueContext; - onCancel?: () => void; -}) { - const now = useNow(1000); +export function InvestigationProgress({ job, now, queue }: { job: Job; now: number; queue?: QueueContext }) { const reason = useQueueReason(job, queue); const progress = analysisProgress(job); - const fraction = analysisFraction(progress); - const samples = useProgressSamples(progress); - const pace = analysisPace(samples, now); - const percent = Math.round(fraction * 100); + const percent = Math.round(analysisFraction(progress) * 100); + const { secondsLeft } = analysisPace(useProgressSamples(progress), now); const queued = progress.step < 0; - const counts = analysisStages(job); - const durations = stageDurations(samples, job.created_at, now); - const filled = Math.round(fraction * blocks); - const stats = [ - ["eta", remainingLabel(pace.secondsLeft)], - ["rate", pace.perMinute === null ? "–" : `${Math.round(pace.perMinute)}/min`], - ["elapsed", analysisElapsed(job.created_at, now)], - ]; - return ( -
-
- - {reason ? QUEUED_TITLE[reason.kind] : progress.title} - {!queued && · {progress.detail}} +
+
+ + {queued ? "–" : `${percent}%`} + + + {analysisElapsed(job.created_at, now)} elapsed + {!queued && ` · ${remainingLabel(secondsLeft)} left`} - {onCancel && ( - - )}
-
-
- -
+ {analysisStages(job).map((stage) => ( +
  • - {Array.from({ length: blocks }, (_, index) => ( - +
    - ))} -
    - - {percent}% -
  • -
      - {steps.map((label, index) => { - const state = stageState(index, progress.step); - const { done, total } = counts[index]; - const seconds = durations[index]; - return ( -
    1. - - {label} - - {state === "todo" || !total ? "–" : `${Math.min(done, total)}/${total}`} +
    +
    + {stage.label} + {stage.state !== "todo" && stage.total > 0 && ( + + {stage.done.toLocaleString()} / {stage.total.toLocaleString()} - {seconds === null ? "" : durationText(seconds)} - - ); - })} - -
    - {queued ? ( - - {reason ? : progress.detail} - - ) : ( - stats.map(([key, value]) => ( - - {key} {value} - - )) - )} -
    -
    + )} +
    + + ))} + + {reason && ( +

    + +

    + )}
    ); } diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx index 45f89d3c876..981d8344eca 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx @@ -294,8 +294,12 @@ it("runs with saved settings from Run now without opening setup, then accepts an }); proxy.post.mockResolvedValue(lens); const user = userEvent.setup(); + const runNow = async () => { + await user.click(await screen.findByRole("button", { name: "Investigation actions" })); + await user.click(await screen.findByRole("menuitem", { name: "Run now" })); + }; renderWithProviders(); - await user.click(await screen.findByRole("button", { name: "Run now" })); + await runNow(); const choices = await screen.findByRole("dialog", { name: "Run now" }); expect(within(choices).getByRole("button", { name: "Since last run" })).toHaveAttribute("aria-pressed", "true"); await user.click(within(choices).getByRole("button", { name: "Run now" })); @@ -303,7 +307,7 @@ it("runs with saved settings from Run now without opening setup, then accepts an await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); proxy.post.mockClear(); - await user.click(screen.getByRole("button", { name: "Run now" })); + await runNow(); const custom = await screen.findByRole("dialog", { name: "Run now" }); fireEvent.change(within(custom).getByRole("combobox", { name: "Agent" }), { target: { value: "billing" } }); await user.click(within(custom).getByRole("button", { name: "Last 24h" })); @@ -770,6 +774,22 @@ it("keeps partial findings visible and shows how many analysis tasks failed", as expect(screen.getByLabelText("Investigation error")).toBeVisible(); }); +it("sends the report's Review issues action to the open issues of that run", async () => { + window.history.replaceState({}, "", "/lens/?lens=lens&kind=pattern&finding_status=resolved"); + testQueryClient.clear(); + proxy.get.mockImplementation(async (path) => { + if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true }; + if (path === "/lens/lens/runs") return lens.jobs; + return { data: [] }; + }); + const user = userEvent.setup(); + renderWithProviders(); + const report = within(await screen.findByRole("region", { name: "Run report" })); + expect(screen.queryByRole("button", { name: issue.title })).not.toBeInTheDocument(); + await user.click(report.getByRole("button", { name: "Review 1 issue" })); + expect(await screen.findByRole("button", { name: new RegExp(issue.title) })).toBeVisible(); +}); + it("keeps a finding open to retry when its update fails", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); @@ -810,7 +830,7 @@ it("pauses monitoring from the detail menu by saving the investigation with moni expect(sentBody(proxy.put, "/lens/lens")).toEqual([{ ...watching.settings, enabled: false }]); }); -it("cancels the running job from the progress banner", async () => { +it("stops the running job from the run report's primary action", async () => { testQueryClient.clear(); const running = { ...lens.jobs[0], id: "live", status: "running" as const, stage: "Reading executions" }; proxy.get.mockImplementation(async (path) => { @@ -822,7 +842,25 @@ it("cancels the running job from the progress banner", async () => { }); const user = userEvent.setup(); renderWithProviders(); - await user.click(await screen.findByRole("button", { name: "Cancel" })); + await user.click(await screen.findByRole("button", { name: "Stop run" })); + await waitFor(() => expect(proxy.post).toHaveBeenCalledWith("/lens/lens/cancel", expect.anything())); +}); + +it("keeps Stop run on an older run while another run is active", async () => { + testQueryClient.clear(); + window.history.replaceState({}, "", `/lens/?lens=lens&run=${lens.jobs[0].id}`); + const running = { ...lens.jobs[0], id: "live", status: "running" as const, stage: "Reading executions" }; + proxy.get.mockImplementation(async (path) => { + if (path.endsWith("/reviews")) return { reviews: [], reviewed: 0 }; + if (path === "/lens") + return { lenses: [{ ...lens, jobs: [running, lens.jobs[0]] }], workers: [], tracing_enabled: true }; + if (path === "/lens/lens/runs") return [running, lens.jobs[0]]; + return { data: [] }; + }); + const user = userEvent.setup(); + renderWithProviders(); + const report = await screen.findByRole("region", { name: "Run report" }); + await user.click(await within(report).findByRole("button", { name: "Stop run" })); await waitFor(() => expect(proxy.post).toHaveBeenCalledWith("/lens/lens/cancel", expect.anything())); }); @@ -923,7 +961,8 @@ it("lists recorded agents in Run now and runs the one picked from the list", asy proxy.post.mockResolvedValue(lens); const user = userEvent.setup(); renderWithProviders(); - await user.click(await screen.findByRole("button", { name: "Run now" })); + await user.click(await screen.findByRole("button", { name: "Investigation actions" })); + await user.click(await screen.findByRole("menuitem", { name: "Run now" })); const dialog = within(await screen.findByRole("dialog", { name: "Run now" })); await user.click(dialog.getByRole("combobox", { name: "Agent" })); expect(await screen.findByRole("option", { name: "support-bot" })).toBeVisible(); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.tsx index aa9809e9a55..bd74a5a17a4 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.tsx @@ -170,6 +170,7 @@ export function InvestigationsView({ readOnly = false }: InvestigationsViewProps onEnableMonitoring={() => openDialog("monitoring")} onCancelRun={() => void actions.cancelRun(row.lens)} onRunNow={() => openDialog("run_now")} + onConnectWorker={() => setTab("settings")} onReviewFinding={(owned, reviewStatus, reason) => void actions.review(owned.lens, owned.finding, reviewStatus, reason) } diff --git a/ui/litellm-dashboard/src/components/lens/investigations/StepFeed.tsx b/ui/litellm-dashboard/src/components/lens/investigations/StepFeed.tsx new file mode 100644 index 00000000000..a085f5b22b3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/investigations/StepFeed.tsx @@ -0,0 +1,41 @@ +"use client"; + +import { cn } from "@/lib/cva.config"; + +import { stepLine } from "../model/inbox"; +import type { Job } from "../model/types"; + +const STEP_MARK = { model: "✓", stage: "▸", error: "!" } as const; +const STEP_TONE = { + model: "text-foreground", + stage: "text-info", + error: "text-destructive", +} as const; + +export function StepFeed({ job }: { job: Job }) { + const steps = job.steps ?? []; + return ( +
      + {steps.length === 0 &&
    1. Waiting for the worker to start…
    2. } + {[...steps].reverse().map((step, index) => ( +
    3. + + + {new Date(step.at).toLocaleTimeString(undefined, { hour12: false })} + + {stepLine(step)} + {step.model} +
    4. + ))} +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.test.tsx index ac6f10bbf5f..b7fe248d8f4 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.test.tsx @@ -53,7 +53,8 @@ describe("InvestigationActions", () => { it("reports each menu choice as an intent without touching the API", async () => { const user = userEvent.setup(); const intents = renderActions(); - await user.click(screen.getByRole("button", { name: "Run now" })); + await user.click(screen.getByRole("button", { name: "Investigation actions" })); + await user.click(await screen.findByRole("menuitem", { name: "Run now" })); expect(intents.onRunNow).toHaveBeenCalledOnce(); await user.click(screen.getByRole("button", { name: "Investigation actions" })); await user.click(await screen.findByRole("menuitem", { name: "Pause monitoring" })); @@ -77,7 +78,7 @@ describe("InvestigationActions", () => { expect(intents.onPause).not.toHaveBeenCalled(); }); - it("blocks a second run while one is queued or running", () => { + it("blocks a second run while one is queued or running", async () => { const job = { id: "job", status: "queued" as const, @@ -95,7 +96,9 @@ describe("InvestigationActions", () => { steps: [], findings: [], }; + const user = userEvent.setup(); renderActions({ lens: { ...lens, jobs: [job] } }); - expect(screen.getByRole("button", { name: "Run now" })).toBeDisabled(); + await user.click(screen.getByRole("button", { name: "Investigation actions" })); + expect(await screen.findByRole("menuitem", { name: "Run now" })).toHaveAttribute("aria-disabled", "true"); }); }); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.tsx b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.tsx index fa00ea0607e..a74e0f32607 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.tsx @@ -18,9 +18,10 @@ export interface InvestigationIntents { readonly onEnableMonitoring: () => void; readonly onCancelRun: () => void; readonly onRunNow: () => void; + readonly onConnectWorker: () => void; } -export type InvestigationActionsProps = Omit & { +export type InvestigationActionsProps = Omit & { readonly lens: Lens; readonly ready: boolean; readonly busy: boolean; @@ -37,37 +38,35 @@ export function InvestigationActions({ onRunNow, }: InvestigationActionsProps) { return ( -
    - - }> - - - - - - Edit investigation + + }> + + + + + + Run now + + + + Edit investigation + + + + Duplicate + + {lens.settings.enabled ? ( + + + Pause monitoring - - - Duplicate + ) : ( + + + Enable monitoring - {lens.settings.enabled ? ( - - - Pause monitoring - - ) : ( - - - Enable monitoring - - )} - - - -
    + )} + + ); } diff --git a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationDetail.tsx b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationDetail.tsx index 7427263edad..52f0d15a509 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationDetail.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationDetail.tsx @@ -4,18 +4,17 @@ import { Button } from "@/components/ui/button"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { RunsTab } from "./RunsTab"; -import { InvestigationProgress } from "../InvestigationProgress"; import { LiveRunLoader } from "../live/LiveRunLoader"; import type { QueueContext } from "../useQueueReason"; import { liveJob } from "../../model/live"; import { InvestigationSummary } from "./InvestigationSummary"; -import { InvestigationFailure } from "./InvestigationFailure"; -import { scopeLabel, sourceLabels } from "../../model/format"; +import { RunReport, type RunActionHandlers } from "./RunReport"; +import { sourceLabels } from "../../model/format"; +import { mergeFeedback } from "../../model/findings"; import type { OwnedFinding } from "../../model/inbox"; -import { activeJob } from "../../model/status"; import { type Finding, type Lens } from "../../model/types"; import { FindingPanel } from "../FindingDetails"; -import { useSectionRoute } from "../../route"; +import { useFindingFilters, useSectionRoute } from "../../route"; import { useRunSnapshot } from "../useRunSnapshot"; import { InvestigationActions, type InvestigationIntents } from "./InvestigationActions"; @@ -46,31 +45,44 @@ export function InvestigationDetail({ ...intents }: InvestigationDetailProps) { const { section, setSection } = useSectionRoute(); + const { setKind, setStatus } = useFindingFilters(); const snapshot = useRunSnapshot(lens); const { job, batchId, batchSettings, batchFindings, missingSnapshot } = snapshot; - const active = activeJob(lens.jobs); const live = liveJob(lens.jobs); + const runActions: RunActionHandlers = { + run: intents.onRunNow, + retry: intents.onRunNow, + stop: onCancelRun, + raiseBudget: intents.onEdit, + connectWorker: intents.onConnectWorker, + monitor: intents.onEnableMonitoring, + reviewIssues: () => { + setSection("findings"); + setKind("issue"); + setStatus("open"); + }, + }; return (
    -
    +

    {lens.settings.name}

    -

    - {sourceLabels[lens.settings.source ?? "traces"]} · {scopeLabel(lens.settings)} -

    +
    {!readOnly && }
    - - {active && ( - - )} + } + actions={readOnly ? undefined : runActions} + queue={queue} + /> {live && ( )} - {job?.error && }
    @@ -89,7 +100,6 @@ export function InvestigationDetail({ {sourceLabels[batchSettings?.source ?? "traces"]} History - {section !== "activity" && }
    {snapshot.error && (

    diff --git a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationFailure.tsx b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationFailure.tsx deleted file mode 100644 index 54943455860..00000000000 --- a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationFailure.tsx +++ /dev/null @@ -1,55 +0,0 @@ -import type { ComponentProps } from "react"; -import { runTime } from "../../model/format"; -import { type Job } from "../../model/types"; -import { failedTaskSummary, isPartial } from "../../model/status"; -import { cn } from "@/lib/cva.config"; - -export type InvestigationFailureProps = ComponentProps<"div"> & { - job: Job; - connected: boolean; -}; - -export function InvestigationFailure({ job, connected, className, ...props }: InvestigationFailureProps) { - const partial = isPartial(job); - return ( -

    -

    - {partial ? "Partial results" : "This investigation did not finish"} -

    - {partial &&

    {failedTaskSummary(job)}. Valid results are preserved.

    } -
    - Run details -
    -          {job.error}
    -        
    -
    -
    -
    Run:
    -
    {job.id}
    -
    -
    -
    Model:
    -
    {job.settings.model}
    -
    -
    -
    Worker:
    -
    {connected ? "Connected now" : "Not connected"}
    -
    -
    -
    Started:
    -
    {runTime(job.created_at)}
    -
    -
    -
    -
    - ); -} diff --git a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationSummary.tsx b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationSummary.tsx index bad16aa0f48..9c06cd97614 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationSummary.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationSummary.tsx @@ -1,49 +1,24 @@ -import type { ComponentProps } from "react"; -import { NextCheck } from "./JobMeta"; -import { lensStatus } from "../../model/status"; -import { runTime } from "../../model/format"; +"use client"; + +import { useNow } from "@/hooks/useNow"; +import { durationLabel, money, scopeLabel, sourceLabels } from "../../model/format"; +import { nextCheckStatus } from "../../model/status"; import { type Lens } from "../../model/types"; -import { cn } from "@/lib/cva.config"; -export type InvestigationSummaryProps = ComponentProps<"div"> & { - lens: Lens; - connected: boolean; -}; - -export function InvestigationSummary({ lens, connected, className, ...props }: InvestigationSummaryProps) { - const lastCompleted = lens.jobs.find((job) => job.status === "completed" && !job.error); - const lastSuccess = lastCompleted?.finished_at ?? lens.last_scan_at; - const spent = lens.budget_month === new Date().toISOString().slice(0, 7) ? lens.spent ?? 0 : 0; +export function InvestigationSummary({ lens }: { lens: Lens }) { + const now = useNow(15000); + const { settings } = lens; + const spent = lens.budget_month === new Date(now).toISOString().slice(0, 7) ? lens.spent ?? 0 : 0; + const parts = [ + scopeLabel(settings), + settings.enabled ? `Every ${durationLabel(settings.interval_minutes)}` : "One-off", + nextCheckStatus(lens, now), + `${money(spent)} of ${money(settings.monthly_budget ?? 100)} this month`, + ].filter(Boolean); return ( -
    - - Latest run:{" "} - - {lensStatus(lens, connected)} - - - - Last full completion: {lastSuccess ? runTime(lastSuccess) : "Not yet"} - - - This month:{" "} - - ${spent.toFixed(3)} / ${lens.settings.monthly_budget ?? 100} - - - {lens.settings.enabled && ( - - Monitoring every {lens.settings.interval_minutes} minutes - - - )} -
    +

    + {sourceLabels[settings.source ?? "traces"]} + {parts.map((part) => ` · ${part}`)} +

    ); } diff --git a/ui/litellm-dashboard/src/components/lens/investigations/detail/JobMeta.tsx b/ui/litellm-dashboard/src/components/lens/investigations/detail/JobMeta.tsx index a846b68df3d..4ee14eb43aa 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/detail/JobMeta.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/detail/JobMeta.tsx @@ -1,16 +1,7 @@ "use client"; -import { useNow } from "@/hooks/useNow"; import { analysisElapsed } from "../../model/progress"; -import { nextCheckStatus } from "../../model/status"; -import type { Job, Lens } from "../../model/types"; - -export function NextCheck({ lens }: { lens: Lens }) { - const now = useNow(15000); - const label = nextCheckStatus(lens, now); - if (!label) return null; - return

    {label}

    ; -} +import type { Job } from "../../model/types"; export function ScanDuration({ job }: { job: Job }) { if (!job.finished_at) return null; diff --git a/ui/litellm-dashboard/src/components/lens/investigations/detail/RunReport.tsx b/ui/litellm-dashboard/src/components/lens/investigations/detail/RunReport.tsx new file mode 100644 index 00000000000..6ac47c8e846 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/investigations/detail/RunReport.tsx @@ -0,0 +1,321 @@ +"use client"; + +import { + ChevronRight, + CircleStop, + ListChecks, + Play, + Plug, + RefreshCw, + Repeat, + Wallet, + type LucideIcon, +} from "lucide-react"; +import type { ComponentProps, ReactNode } from "react"; + +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/cva.config"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { useNow } from "@/hooks/useNow"; + +import { analysisElapsed } from "../../model/progress"; +import { money } from "../../model/format"; +import { modelsUsed, shortTime, windowLabel } from "../../model/inbox"; +import { + NEXT_ACTION, + openIssues, + runSituation, + type RunAction, + type RunSituation, + type SituationInput, +} from "../../model/runSituation"; +import type { Finding, Job, Lens } from "../../model/types"; +import { activeJob, failedTaskSummary } from "../../model/status"; +import { InvestigationProgress } from "../InvestigationProgress"; +import type { QueueContext } from "../useQueueReason"; +import { StepFeed } from "../StepFeed"; + +export type RunActionHandlers = Record void>; + +export interface RunReportProps { + readonly lens: Lens; + readonly job: Job | undefined; + readonly findings: readonly Finding[] | null | undefined; + readonly connected: boolean; + readonly ready: boolean; + readonly busy: boolean; + readonly picker: ReactNode; + readonly actions?: RunActionHandlers; + readonly queue?: QueueContext; +} + +interface Facts { + readonly runs: string; + readonly found: string; + readonly openIssues: number; +} + +type Tone = "info" | "success" | "warning" | "destructive" | "muted"; + +interface SituationView { + readonly status: string; + readonly tone: Tone; + readonly headline: (facts: Facts) => string; + readonly body: "progress" | "error" | "partial" | null; +} + +const completed = ({ found, runs }: Facts) => + found ? `Found ${found} across ${runs}` : `Nothing found across ${runs}`; + +const SITUATIONS: Record = { + never: { status: "Not run yet", tone: "muted", headline: () => "Run it to get the first report", body: null }, + queued: { + status: "Queued", + tone: "info", + headline: () => "Waiting for an analyzer to pick this up", + body: "progress", + }, + running: { status: "Running", tone: "info", headline: () => "Investigating now", body: "progress" }, + budget: { + status: "Failed", + tone: "destructive", + headline: () => "Stopped: the monthly budget is used up", + body: "error", + }, + offline: { + status: "Failed", + tone: "destructive", + headline: () => "Stopped: no analyzer is connected", + body: "error", + }, + failed: { status: "Failed", tone: "destructive", headline: () => "Stopped before it finished", body: "error" }, + cancelled: { + status: "Cancelled", + tone: "muted", + headline: ({ runs }) => `Cancelled after reviewing ${runs}`, + body: "error", + }, + partial: { status: "Partial results", tone: "warning", headline: completed, body: "partial" }, + unknown: { status: "Completed", tone: "success", headline: ({ runs }) => `Reviewed ${runs}`, body: null }, + issues: { status: "Completed", tone: "success", headline: completed, body: null }, + watching: { status: "Completed", tone: "success", headline: completed, body: null }, + clean: { status: "Completed", tone: "success", headline: completed, body: null }, +}; + +interface ActionView { + readonly label: (facts: Facts) => string; + readonly icon: LucideIcon; + readonly variant: ComponentProps["variant"]; + readonly needsReady: boolean; +} + +const ACTIONS: Record = { + run: { label: () => "Run now", icon: Play, variant: "default", needsReady: true }, + stop: { label: () => "Stop run", icon: CircleStop, variant: "outline", needsReady: false }, + retry: { label: () => "Retry", icon: RefreshCw, variant: "default", needsReady: true }, + raiseBudget: { label: () => "Raise budget", icon: Wallet, variant: "default", needsReady: false }, + connectWorker: { label: () => "Connect worker", icon: Plug, variant: "default", needsReady: false }, + reviewIssues: { + label: ({ openIssues: count }) => `Review ${plural(count, "issue")}`, + icon: ListChecks, + variant: "default", + needsReady: false, + }, + monitor: { label: () => "Monitor this", icon: Repeat, variant: "default", needsReady: true }, +}; + +const TONE_CLASS: Record = { + info: "text-info", + success: "text-success", + warning: "text-warning", + destructive: "text-destructive", + muted: "text-muted-foreground", +}; + +const plural = (count: number, noun: string) => `${count.toLocaleString()} ${noun}${count === 1 ? "" : "s"}`; + +function facts(job: Job | undefined, findings: readonly Finding[] | null | undefined): Facts { + const count = (kind: Finding["kind"], noun: string) => { + const total = findings?.filter((f) => f.kind === kind).length ?? 0; + return total ? plural(total, noun) : ""; + }; + return { + runs: plural(job?.coverage?.screened ?? 0, "run"), + found: [count("issue", "issue"), count("pattern", "pattern")].filter(Boolean).join(" and "), + openIssues: openIssues(findings), + }; +} + +function Stat({ label, value, note }: { label: string; value: string; note?: string }) { + return ( +
    +
    {label}
    +
    {value}
    + {note &&
    {note}
    } +
    + ); +} + +function coverageNote(job: Job): string | undefined { + const { partial = 0, unassessable = 0, inconclusive = 0 } = job.coverage ?? {}; + const notes = [ + partial && `${partial} partial`, + unassessable && `${unassessable} unreadable`, + inconclusive && `${inconclusive} inconclusive`, + ].filter(Boolean); + return notes.length ? notes.join(" · ") : undefined; +} + +const isActive = (job: Job) => job.status === "queued" || job.status === "running"; + +function issueStat(job: Job, findings: readonly Finding[] | null | undefined) { + if (isActive(job)) return { value: "–", note: "When the run finishes" }; + if (job.status !== "completed" || findings == null) return { value: "–" }; + const issues = findings.filter((f) => f.kind === "issue"); + const high = issues.filter((f) => f.priority === "high").length; + return { value: issues.length.toLocaleString(), note: high ? `${high} high priority` : undefined }; +} + +function RunStats({ job, findings, now }: { job: Job; findings: readonly Finding[] | null | undefined; now: number }) { + const calls = (job.steps ?? []).filter((step) => step.kind === "model").length; + const { screened = 0, selected = 0 } = job.coverage ?? {}; + const end = job.finished_at ? Date.parse(job.finished_at) : now; + return ( +
    + + + + +
    + ); +} + +function RunFailure({ job, connected }: { job: Job; connected: boolean }) { + return ( +
    + +
    + ); +} + +function RunError({ job, connected }: { job: Job; connected: boolean }) { + return ( + <> +
    +        {job.error}
    +      
    +

    + + Run {job.id} + + + Model {job.settings.model} + + Worker {connected ? "connected now" : "not connected"} +

    + + ); +} + +function RunPartial({ job, connected }: { job: Job; connected: boolean }) { + return ( +
    +

    {failedTaskSummary(job)}. Valid results are preserved.

    +
    + Run details +
    + +
    +
    +
    + ); +} + +function RunLog({ job }: { job: Job }) { + const models = modelsUsed(job.steps ?? []); + return ( + +
    + + + Activity log + + + {job.trigger === "manual" ? "Manual" : "Scheduled"} · {windowLabel(job)} + {models.length > 0 && ` · ${models.join(", ")}`} + +
    + + + +
    + ); +} + +function NextAction({ + action, + facts: known, + ready, + busy, + onClick, +}: { + action: RunAction; + facts: Facts; + ready: boolean; + busy: boolean; + onClick: () => void; +}) { + const { label, icon: Icon, variant, needsReady } = ACTIONS[action]; + return ( + + ); +} + +function nextAction(lens: Lens, job: Job | undefined, situation: RunSituation): RunAction | null { + if (job === undefined || job.id === lens.jobs[0]?.id) return NEXT_ACTION[situation]; + return activeJob(lens.jobs) ? "stop" : null; +} + +export function RunReport({ lens, job, findings, connected, ready, busy, picker, actions, queue }: RunReportProps) { + const now = useNow(job && isActive(job) ? 1000 : 60000); + const input: SituationInput = { lens, job, findings, connected }; + const situation = runSituation(input); + const view = SITUATIONS[situation]; + const known = facts(job, findings); + const action = nextAction(lens, job, situation); + return ( +
    +
    +
    +

    {view.status}

    +

    {view.headline(known)}

    +
    +
    + {job && picker} + {action && actions && ( + + )} +
    +
    + {job && view.body === "progress" && } + {job?.error && view.body === "partial" && } + {job?.error && view.body === "error" && } + {job && } + {job && } +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/model/inbox.ts b/ui/litellm-dashboard/src/components/lens/model/inbox.ts index b1ab117d0b8..ff01a0ad2b6 100644 --- a/ui/litellm-dashboard/src/components/lens/model/inbox.ts +++ b/ui/litellm-dashboard/src/components/lens/model/inbox.ts @@ -155,9 +155,10 @@ const WINDOW_FORMAT: Intl.DateTimeFormatOptions = { minute: "2-digit", }; +export const shortTime = (iso: string) => new Date(iso).toLocaleString(undefined, WINDOW_FORMAT); + export function windowLabel(job: Job): string { - const fmt = (iso: string) => new Date(iso).toLocaleString(undefined, WINDOW_FORMAT); - return `${fmt(job.start)} → ${fmt(job.end)}`; + return `${shortTime(job.start)} → ${shortTime(job.end)}`; } export function inboxFinding(row: InboxRow): Finding { diff --git a/ui/litellm-dashboard/src/components/lens/model/progress.test.ts b/ui/litellm-dashboard/src/components/lens/model/progress.test.ts index 934e85111d8..ab04cff498c 100644 --- a/ui/litellm-dashboard/src/components/lens/model/progress.test.ts +++ b/ui/litellm-dashboard/src/components/lens/model/progress.test.ts @@ -3,9 +3,9 @@ import { analysisElapsed, analysisProgress, analysisFraction, + analysisStages, analysisPace, remainingLabel, - stageDurations, } from "./progress"; import { type Job } from "./types"; @@ -216,20 +216,27 @@ describe("Analysis pace", () => { }); }); -describe("Stage durations", () => { - const start = Date.parse("2026-09-30T12:00:00Z"); - const createdAt = "2026-09-30T12:00:00Z"; - - it("times finished stages from the transitions it saw and the active stage up to now", () => { - const samples = [ - { at: start + 5000, step: 0, done: 10, fraction: 0.1 }, - { at: start + 124000, step: 1, done: 0, fraction: 0.6 }, - ]; - expect(stageDurations(samples, createdAt, start + 145000)).toEqual([124, 21, null]); +describe("Analysis stages", () => { + it("fills finished stages, fills the active one by its own count, and leaves later ones empty", () => { + const grouping = { + ...job, + stage: "Grouping observations", + coverage: { ...coverage, selected: 40, screened: 40, grouping_batches: 4, grouped_batches: 1 }, + }; + expect(analysisStages(grouping).map(({ state, fill, done, total }) => ({ state, fill, done, total }))).toEqual([ + { state: "done", fill: 1, done: 40, total: 40 }, + { state: "active", fill: 0.25, done: 1, total: 4 }, + { state: "todo", fill: 0, done: 0, total: 0 }, + ]); }); - it("does not guess when a stage started before the page was opened", () => { - const samples = [{ at: start + 90000, step: 1, done: 2, fraction: 0.7 }]; - expect(stageDurations(samples, createdAt, start + 100000)).toEqual([null, null, null]); + it("weights the stages so the bar agrees with the overall percentage", () => { + const grouping = { + ...job, + stage: "Grouping observations", + coverage: { ...coverage, selected: 40, screened: 40, grouping_batches: 4, grouped_batches: 1 }, + }; + const covered = analysisStages(grouping).reduce((sum, stage) => sum + stage.weight * stage.fill, 0); + expect(covered).toBeCloseTo(analysisFraction(analysisProgress(grouping))); }); }); diff --git a/ui/litellm-dashboard/src/components/lens/model/progress.ts b/ui/litellm-dashboard/src/components/lens/model/progress.ts index d1a22431701..880e91f780d 100644 --- a/ui/litellm-dashboard/src/components/lens/model/progress.ts +++ b/ui/litellm-dashboard/src/components/lens/model/progress.ts @@ -59,20 +59,47 @@ export function analysisProgress(job: Job) { }; } -export function analysisStages(job: Job): { done: number; total: number }[] { - const { - screened = 0, - selected = 0, - grouped_batches = 0, - grouping_batches = 0, - investigated = 0, - candidates = 0, - } = job.coverage ?? {}; - return [ - { done: screened, total: selected }, - { done: grouped_batches, total: grouping_batches }, - { done: investigated, total: candidates }, - ]; +type Coverage = Job["coverage"]; + +export const ANALYSIS_STAGES = [ + { label: "Review runs", weight: 0.6, done: (c: Coverage) => c.screened, total: (c: Coverage) => c.selected }, + { + label: "Find patterns", + weight: 0.2, + done: (c: Coverage) => c.grouped_batches, + total: (c: Coverage) => c.grouping_batches, + }, + { label: "Check evidence", weight: 0.2, done: (c: Coverage) => c.investigated, total: (c: Coverage) => c.candidates }, +] as const; + +export interface StageProgress { + readonly label: string; + readonly weight: number; + readonly state: "done" | "active" | "todo"; + readonly done: number; + readonly total: number; + readonly fill: number; +} + +function stageState(index: number, current: number): StageProgress["state"] { + if (index < current) return "done"; + return index === current ? "active" : "todo"; +} + +function stageFill(state: StageProgress["state"], done: number, total: number): number { + if (state === "done") return 1; + if (state === "todo" || !total) return 0; + return Math.min(1, done / total); +} + +export function analysisStages(job: Job): readonly StageProgress[] { + const step = analysisProgress(job).step; + return ANALYSIS_STAGES.map((stage, index) => { + const state = stageState(index, step); + const done = job.coverage ? stage.done(job.coverage) : 0; + const total = job.coverage ? stage.total(job.coverage) : 0; + return { label: stage.label, weight: stage.weight, state, done, total, fill: stageFill(state, done, total) }; + }); } export interface ProgressSample { @@ -82,16 +109,14 @@ export interface ProgressSample { fraction: number; } -const stageWeights = [0.6, 0.2, 0.2]; - export function analysisFraction({ step, done, total, }: Pick, "step" | "done" | "total">): number { if (step < 0) return 0; - const before = stageWeights.slice(0, step).reduce((sum, weight) => sum + weight, 0); - return before + stageWeights[step] * (total ? Math.min(1, done / total) : 0); + const before = ANALYSIS_STAGES.slice(0, step).reduce((sum, stage) => sum + stage.weight, 0); + return before + ANALYSIS_STAGES[step].weight * (total ? Math.min(1, done / total) : 0); } function windowStart(samples: readonly ProgressSample[], now: number): ProgressSample | undefined { @@ -115,22 +140,6 @@ export function analysisPace(samples: readonly ProgressSample[], now: number) { return { perMinute, secondsLeft }; } -export function stageDurations(samples: readonly ProgressSample[], createdAt: string, now: number): (number | null)[] { - const current = samples.at(-1)?.step ?? -1; - const starts = [0, 1, 2].map((stage) => { - if (stage === 0) return Date.parse(createdAt); - const entered = samples.findIndex( - (sample, index) => index > 0 && sample.step >= stage && samples[index - 1].step < stage, - ); - return entered < 0 ? null : samples[entered].at; - }); - return starts.map((start, stage) => { - if (start === null || stage > current) return null; - const end = stage === current ? now : starts[stage + 1]; - return end === null ? null : Math.max(0, Math.floor((end - start) / 1000)); - }); -} - export function remainingLabel(seconds: number | null): string { if (seconds === null) return "estimating"; if (seconds < 60) return "<1m"; diff --git a/ui/litellm-dashboard/src/components/lens/model/runSituation.test.ts b/ui/litellm-dashboard/src/components/lens/model/runSituation.test.ts new file mode 100644 index 00000000000..5e3a38d1c89 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/model/runSituation.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; +import { NEXT_ACTION, runSituation, type SituationInput } from "./runSituation"; +import type { Finding, Job, Lens } from "./types"; + +const settings: Lens["settings"] = { + context: "", + source: "traces", + lookback_hours: 24, + service: "", + agent_name: "", + filters: [], + interval_minutes: 60, + sample_size: 100, + sample_percent: 100, + concurrency: 8, + team_id: "", + execution_ids: [], + monthly_budget: 20, + name: "Release reviews", + model: "analysis", + enabled: false, + checks: [], +}; +const month = new Date().toISOString().slice(0, 7); +const lens: Lens = { + version: 0, + spent: 1, + id: "lens", + scope: { all_teams: true, api_key_hash: "", team_id: "" }, + settings, + revision: 1, + created_at: "2026-09-30T10:00:00Z", + next_run_at: "2026-09-30T10:00:00Z", + budget_month: month, + findings: [], + jobs: [], +}; +const job: Job = { + id: "run", + assessments: [], + steps: [], + reviews: [], + reviewed: 0, + reading: [], + activities: [], + trigger: "manual", + attempts: 1, + error: "", + cost: 0, + coverage: { + eligible: 10, + selected: 10, + screened: 10, + investigated: 0, + inconclusive: 0, + grouping_batches: 0, + grouped_batches: 0, + candidates: 0, + partial: 0, + unassessable: 0, + failed_tasks: 0, + }, + status: "completed", + stage: "Complete", + created_at: "2026-09-30T10:00:00Z", + start: "2026-09-29T10:00:00Z", + end: "2026-09-30T10:00:00Z", + settings, + revision: 1, +}; +const finding = (kind: Finding["kind"], status: Finding["status"]): Finding => ({ + id: `${kind}-${status}`, + check_id: "check", + title: "", + description: "", + reason: "", + suggestion: "", + limitation: "", + kind, + priority: "high", + status, + revision: 1, + first_seen: job.created_at, + last_seen: job.created_at, + occurrences: [], + evidence: [], +}); + +const input = (overrides: Partial & { jobPatch?: Partial }): SituationInput => { + const { jobPatch, ...rest } = overrides; + return { lens, job: { ...job, ...jobPatch }, findings: [], connected: true, ...rest }; +}; +const failed = { status: "failed", error: "Model request failed" } as const; +const watching = { ...lens, settings: { ...settings, enabled: true } }; + +describe("runSituation", () => { + it.each([ + ["never", input({ job: undefined }), "run"], + ["queued", input({ jobPatch: { status: "queued" } }), "stop"], + ["running", input({ jobPatch: { status: "running" } }), "stop"], + ["budget", input({ jobPatch: failed, lens: { ...lens, spent: 20 } }), "raiseBudget"], + ["budget", input({ jobPatch: { ...failed, error: "Monthly lens budget reached" } }), "raiseBudget"], + ["offline", input({ jobPatch: failed, connected: false }), "connectWorker"], + ["failed", input({ jobPatch: failed }), "retry"], + ["failed", input({ jobPatch: { ...failed, error: "Could not reserve analysis budget" } }), "retry"], + ["cancelled", input({ jobPatch: { status: "cancelled" } }), "retry"], + ["partial", input({ jobPatch: { error: "Result validation failed after 3 retries" } }), "retry"], + ["unknown", input({ findings: null }), null], + ["issues", input({ lens: watching, findings: [finding("issue", "open")] }), "reviewIssues"], + ["watching", input({ lens: watching, findings: [finding("issue", "resolved")] }), null], + ["clean", input({ findings: [finding("pattern", "open")] }), "monitor"], + ] as const)("reads %s and offers %s", (expected, situationInput, action) => { + const situation = runSituation(situationInput); + expect(situation).toBe(expected); + expect(NEXT_ACTION[situation]).toBe(action); + }); + + it("puts a running job ahead of a spent budget, so Stop stays reachable", () => { + expect(runSituation(input({ jobPatch: { status: "running" }, lens: { ...lens, spent: 99 } }))).toBe("running"); + }); + + it("ignores spend from an earlier month", () => { + expect(runSituation(input({ jobPatch: failed, lens: { ...lens, spent: 99, budget_month: "1999-01" } }))).toBe( + "failed", + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/model/runSituation.ts b/ui/litellm-dashboard/src/components/lens/model/runSituation.ts new file mode 100644 index 00000000000..f595a92e8cf --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/model/runSituation.ts @@ -0,0 +1,65 @@ +import { budgetReached, isPartial } from "./status"; +import type { Finding, Job, Lens } from "./types"; + +export interface SituationInput { + readonly lens: Lens; + readonly job: Job | undefined; + readonly findings: readonly Finding[] | null | undefined; + readonly connected: boolean; +} + +export type RunSituation = + | "never" + | "queued" + | "running" + | "budget" + | "offline" + | "failed" + | "cancelled" + | "partial" + | "unknown" + | "issues" + | "watching" + | "clean"; + +export type RunAction = "run" | "stop" | "retry" | "raiseBudget" | "connectWorker" | "reviewIssues" | "monitor"; + +const failed = (job: Job | undefined) => job?.status === "failed"; + +export const openIssues = (findings: readonly Finding[] | null | undefined) => + findings?.filter((finding) => finding.kind === "issue" && finding.status === "open").length ?? 0; + +// First match wins, so the order is the precedence between overlapping situations +const RULES: readonly (readonly [RunSituation, (input: SituationInput) => boolean])[] = [ + ["never", ({ job }) => !job], + ["queued", ({ job }) => job?.status === "queued"], + ["running", ({ job }) => job?.status === "running"], + ["budget", ({ job, lens }) => failed(job) && (budgetReached(lens) || /budget reached/i.test(job?.error ?? ""))], + ["offline", ({ job, connected }) => failed(job) && !connected], + ["failed", ({ job }) => failed(job)], + ["cancelled", ({ job }) => job?.status === "cancelled"], + ["partial", ({ job }) => job !== undefined && isPartial(job)], + ["unknown", ({ findings }) => findings == null], + ["issues", ({ findings }) => openIssues(findings) > 0], + ["watching", ({ lens }) => lens.settings.enabled], + ["clean", () => true], +]; + +export function runSituation(input: SituationInput): RunSituation { + return RULES.find(([, matches]) => matches(input))?.[0] ?? "clean"; +} + +export const NEXT_ACTION: Record = { + never: "run", + queued: "stop", + running: "stop", + budget: "raiseBudget", + offline: "connectWorker", + failed: "retry", + cancelled: "retry", + partial: "retry", + unknown: null, + issues: "reviewIssues", + watching: null, + clean: "monitor", +}; diff --git a/ui/litellm-dashboard/src/components/lens/model/status.ts b/ui/litellm-dashboard/src/components/lens/model/status.ts index 82541954558..741dedfb6b0 100644 --- a/ui/litellm-dashboard/src/components/lens/model/status.ts +++ b/ui/litellm-dashboard/src/components/lens/model/status.ts @@ -120,11 +120,15 @@ export function listPollInterval(list: LensList | undefined, settingsOpen: boole return running || (settingsOpen && !connected) ? 2000 : 10000; } +export function budgetReached(lens: Lens, now = new Date()): boolean { + const spent = lens.budget_month === now.toISOString().slice(0, 7) ? lens.spent ?? 0 : 0; + return spent >= (lens.settings.monthly_budget ?? 100); +} + export function lensStatus(lens: Lens, connected: boolean): string { const active = lens.jobs?.find((job) => ["queued", "running"].includes(job.status ?? "")); if (active) return connected ? active.stage ?? "Queued" : "Waiting for analyzer"; - const spent = lens.budget_month === new Date().toISOString().slice(0, 7) ? lens.spent ?? 0 : 0; - if (spent >= (lens.settings.monthly_budget ?? 100)) return "Budget reached"; + if (budgetReached(lens)) return "Budget reached"; const latest = lens.jobs?.[0]; return latest ? runStatus(latest) : "Ready"; }