From 13f6e7af03787ae542a1ac583e2975ca3fa64853 Mon Sep 17 00:00:00 2001 From: Tin Date: Fri, 6 Nov 2026 16:05:08 +0800 Subject: [PATCH] feat(ui): add workflow run history and typed trace inspector --- .../workflows/run-preview/page.tsx | 7 + .../src/components/workflows/runs/README.md | 118 +++++++++ .../workflows/runs/RunStatusBadge.tsx | 21 ++ .../workflows/runs/TracePayload.tsx | 17 ++ .../WorkflowRunDebugger.integration.test.tsx | 207 ++++++++++++++++ .../workflows/runs/WorkflowRunDebugger.tsx | 147 ++++++++++++ .../workflows/runs/WorkflowRunHistory.tsx | 160 +++++++++++++ .../workflows/runs/WorkflowRunInspector.tsx | 170 +++++++++++++ .../runs/WorkflowStepSpendDetails.tsx | 37 +++ .../src/components/workflows/runs/api.test.ts | 119 +++++++++ .../src/components/workflows/runs/api.ts | 82 +++++++ .../components/workflows/runs/contracts.ts | 107 +++++++++ .../src/components/workflows/runs/index.ts | 20 ++ .../components/workflows/runs/presentation.ts | 54 +++++ .../workflow-runs/WorkflowRunDemo.tsx | 44 ++++ .../tests/fixtures/workflow-runs/data.ts | 225 ++++++++++++++++++ .../fixtures/workflow-runs/serve-shell.mjs | 118 +++++++++ 17 files changed, 1653 insertions(+) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/workflows/run-preview/page.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/README.md create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/RunStatusBadge.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/TracePayload.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunHistory.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunInspector.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/WorkflowStepSpendDetails.tsx create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/api.test.ts create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/api.ts create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/contracts.ts create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/index.ts create mode 100644 ui/litellm-dashboard/src/components/workflows/runs/presentation.ts create mode 100644 ui/litellm-dashboard/tests/fixtures/workflow-runs/WorkflowRunDemo.tsx create mode 100644 ui/litellm-dashboard/tests/fixtures/workflow-runs/data.ts create mode 100644 ui/litellm-dashboard/tests/fixtures/workflow-runs/serve-shell.mjs diff --git a/ui/litellm-dashboard/src/app/(dashboard)/workflows/run-preview/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/workflows/run-preview/page.tsx new file mode 100644 index 00000000000..afb0a498937 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/workflows/run-preview/page.tsx @@ -0,0 +1,7 @@ +import { notFound } from "next/navigation"; + +export default async function WorkflowRunPreviewPage() { + if (process.env.NODE_ENV !== "development") notFound(); + const { default: WorkflowRunDemo } = await import("../../../../../tests/fixtures/workflow-runs/WorkflowRunDemo"); + return ; +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/README.md b/ui/litellm-dashboard/src/components/workflows/runs/README.md new file mode 100644 index 00000000000..f8ace445b8a --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/README.md @@ -0,0 +1,118 @@ +# Workflow run debugging + +Reusable history and trace components for the workflow page. The development preview uses synthetic API responses and does not execute or bill workflows + +## Public interfaces + +Import from `@/components/workflows/runs` + +| Export | Purpose | +| ----------------------- | ------------------------------------------------------------ | +| `createWorkflowRunApi` | Authenticated HTTP adapter with runtime response validation | +| `WorkflowRunHistory` | Workflow-scoped history, status filter and cursor pagination | +| `WorkflowRunInspector` | Controlled presentation of a trace and its pinned definition | +| `WorkflowRunDebugger` | History, selected-run queries, polling and graph callbacks | +| `WorkflowRunDataSource` | Replaceable run-list, trace and pinned-definition boundary | +| `WorkflowRunGraphState` | Pinned definition, all step executions and selected step | +| `WorkflowStepSelection` | `runId`, `snapshotId`, `stepId` and `nodeId` | + +The barrel also exports request and response types from `contracts.ts`. Keep one data-source instance per backend, team and authenticated session. Recreate it when any of those change. Its opaque `cacheKey` scopes query caches and must never contain a bearer token + +```tsx +const dataSource = useMemo(() => createWorkflowRunApi({ accessToken, teamId }), [accessToken, teamId, backendIdentity]); + +; +``` + +The owning page controls the selected run and any URL state through nuqs. Clear its selected run when switching workflows. Local filters and pagination reset when the workflow, data source or page size changes. `selectedStepId` and `onSelectStep` support selection controlled by the graph. Without a controlled step, the inspector initially selects the failed step, then the first reported step + +## Confirmed trace contract + +`RunSteps` and `StepTrace` match `src/liteflow/client.py` at LiteFlow PR #5 commit `27bcbce4e84fb73b1d4bdf2fa108df53e9b50c2d` + +`GET /v1/workflows/runs/{run_id}/steps` returns `run_id`, `snapshot_id`, `state`, `cancel_requested` and `steps` + +Each step preserves `step_id`, `run_id`, `node_id`, `state`, JSON `input` and `output`, `attempts`, `deadline`, `started_at`, nullable `finished_at`, nullable `error`, `prompt_tokens`, `completion_tokens`, nullable `response_cost` and nullable `request_id` + +Step IDs must be unique within a run. Multiple executions may reference the same node. The graph callback retains every execution rather than using node IDs as step keys. Null, false, zero and empty strings remain distinct JSON values. Unknown costs show “Not reported”. Aggregate cost is labeled “Reported step cost” and marked partial when any step has an unknown cost + +## Provisional gateway assumptions + +Only `api.ts` understands the provisional wire responses + +```text +GET /v1/workflows/runs + team_id= + workflow_id= + after= + status= + limit= + +{ + "runs": [{ + "run_id": "...", + "workflow_id": "...", + "snapshot_id": "...", + "status": "completed", + "created_at": "2026-11-06T08:30:00Z" + }], + "next_cursor": null +} +``` + +```text +GET /v1/workflows/runs/{run_id}/definition + +{ + "workflow_id": "...", + "snapshot_id": "...", + "bundle": { "contract_version": "1", "nodes": [] } +} +``` + +The adapter rejects mismatched workflow, run and snapshot identities. The pinned definition is queried only after the trace supplies its snapshot ID. No current-draft endpoint is used as a fallback. A missing definition leaves the trace available and passes a null definition to the graph + +The first history page refreshes every five seconds. Selected pending and running traces refresh every two seconds, stopping at terminal states. Requests receive React Query abort signals. Selection and cache keys isolate late responses from previous runs or sessions. Poll failures expose a retry action and label displayed data as the last successful response + +These reads can be backed by the gateway's existing workflow run and event storage. The frontend neither reads SpendLogs for history nor maintains a separate persistent store + +## Exact spend details + +“View spend record” appears only when the step has a `request_id`. Opening it uses the dashboard's existing `useLogDetails` authorization flow with that exact ID and the step's original timestamp. The endpoint is `/spend/logs/ui/{request_id}?start_date=...` + +This avoids the Logs page's current date-window search and performs no spend aggregation or billing writes. The payload returned by the authorized detail endpoint is shown as JSON. Permission, retention and missing-log failures leave workflow trace inspection available + +## Local fixture verification + +From `ui/litellm-dashboard`, run these commands in separate terminals + +```sh +../../scripts/with_dashboard_node.sh node tests/fixtures/workflow-runs/serve-shell.mjs +``` + +```sh +LENS_DEV_PROXY_URL=http://127.0.0.1:3378 NEXT_PUBLIC_USE_REWRITES=true \ + ../../scripts/with_dashboard_node.sh npm run dev:webpack -- \ + --hostname 127.0.0.1 --port 3377 +``` + +Open `http://workflow-runs.localhost:3377/__workflow_run_fixture_login`. The preview is development-only, served inside the real dashboard shell, and labeled “Fixture preview”. Its local fixture credential is not a gateway credential + +Select the completed, running and failed runs. Inspect Python values, a failed step with two attempts and an LLM spend record. Filter by status, navigate to the second history page, then inspect two distinct executions referencing `node_1`. The pending fixture has no reported steps. The scalar Python fixture deliberately exercises JSON false and zero as rendering edge cases + +The preview serves run and definition responses over HTTP through `createWorkflowRunApi`. Active fixtures remain active until changed by the fixture server. There is no simulated completion timer. The integration test separately verifies a gateway response transitioning a run to completion and stopping polling + +```sh +../../scripts/with_dashboard_node.sh npx vitest run \ + src/components/workflows/runs/api.test.ts \ + src/components/workflows/runs/WorkflowRunDebugger.integration.test.tsx +``` + +Stop both local processes after verification. This preview does not establish live gateway integration, real provider execution or authorization against the backend under development diff --git a/ui/litellm-dashboard/src/components/workflows/runs/RunStatusBadge.tsx b/ui/litellm-dashboard/src/components/workflows/runs/RunStatusBadge.tsx new file mode 100644 index 00000000000..54a8763ad69 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/RunStatusBadge.tsx @@ -0,0 +1,21 @@ +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/cva.config"; +import type { RunState } from "./contracts"; +import { stateLabels } from "./presentation"; + +const stateColors: Record = { + pending: "bg-muted text-muted-foreground", + running: "bg-info/10 text-info", + completed: "bg-success/10 text-success", + failed: "bg-destructive/10 text-destructive", + cancelled: "bg-muted text-muted-foreground", +}; + +export function RunStatusBadge({ state }: { state: RunState }) { + return ( + + + {stateLabels[state]} + + ); +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/TracePayload.tsx b/ui/litellm-dashboard/src/components/workflows/runs/TracePayload.tsx new file mode 100644 index 00000000000..5acc1194c4a --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/TracePayload.tsx @@ -0,0 +1,17 @@ +import type { JsonValue } from "./contracts"; + +export function TracePayload({ label, value }: { label: string; value: JsonValue }) { + const type = value === null ? "null" : typeof value; + const typeLabel = Array.isArray(value) ? "array" : type; + return ( +
+
+

{label}

+ {typeLabel} +
+
+        {JSON.stringify(value, null, 2)}
+      
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.integration.test.tsx b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.integration.test.tsx new file mode 100644 index 00000000000..889a59a0e74 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.integration.test.tsx @@ -0,0 +1,207 @@ +import { useState } from "react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { renderWithProviders, testQueryClient, chooseSelectOption } from "../../../../tests/test-utils"; +import type { RunSteps, WorkflowRunDataSource, WorkflowRunGraphState } from "./contracts"; +import { + createWorkflowRunFixtureSource, + fixtureDefinitions, + fixtureTraces, + fixtureWorkflowId, +} from "../../../../tests/fixtures/workflow-runs/data"; +import { WorkflowRunDebugger } from "./WorkflowRunDebugger"; + +function Harness({ + source, + initialRun = null, + graphState, +}: { + source: WorkflowRunDataSource; + initialRun?: string | null; + graphState?: (state: WorkflowRunGraphState | null) => void; +}) { + const [runId, setRunId] = useState(initialRun); + return ( + + ); +} + +beforeEach(() => testQueryClient.clear()); +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +describe("workflow run debugging", () => { + it("paginates workflow history and resets the cursor when status changes", async () => { + const source = createWorkflowRunFixtureSource(); + const listRuns = vi.fn(source.listRuns); + renderWithProviders(); + expect(await screen.findByRole("button", { name: "run-fixture-completed" })).toBeVisible(); + fireEvent.click(screen.getByRole("button", { name: "Next" })); + expect(await screen.findByRole("button", { name: "run-fixture-failed" })).toBeVisible(); + const user = userEvent.setup(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter runs by status" }), "Failed"); + await waitFor(() => + expect(listRuns).toHaveBeenLastCalledWith( + expect.objectContaining({ workflowId: fixtureWorkflowId, status: "failed", cursor: null }), + ), + ); + expect(screen.getByText("Page 1")).toBeVisible(); + expect(screen.getByRole("button", { name: "Previous" })).toBeDisabled(); + expect(screen.queryByRole("button", { name: "run-fixture-completed" })).not.toBeInTheDocument(); + }); + + it("shows typed Python values without a spend action and emits the pinned definition", async () => { + const graphState = vi.fn<(state: WorkflowRunGraphState | null) => void>(); + renderWithProviders( + , + ); + fireEvent.click(await screen.findByRole("button", { name: "Inspect step 2 node_2" })); + const details = screen.getByRole("region", { name: "Step details" }); + expect(within(details).getByLabelText("Input JSON")).toHaveTextContent("false"); + expect(within(details).getByLabelText("Output JSON")).toHaveTextContent("0"); + expect(within(details).getByText("$0.000000")).toBeVisible(); + expect(within(details).getByText("25 ms")).toBeVisible(); + expect(within(details).queryByRole("button", { name: "View spend record" })).not.toBeInTheDocument(); + await waitFor(() => + expect(graphState).toHaveBeenLastCalledWith( + expect.objectContaining({ + definition: fixtureDefinitions["snapshot-fixture-v1"], + selection: { + runId: "run-fixture-completed", + snapshotId: "snapshot-fixture-v1", + stepId: "step-completed-2", + nodeId: "node_2", + }, + }), + ), + ); + }); + + it("retains independent selection for two executions of one node", async () => { + const graphState = vi.fn<(state: WorkflowRunGraphState | null) => void>(); + renderWithProviders( + , + ); + fireEvent.click(await screen.findByRole("button", { name: "Inspect step 2 node_1" })); + expect(screen.getByLabelText("Output JSON")).toHaveTextContent('"Second execution"'); + expect(graphState).toHaveBeenLastCalledWith( + expect.objectContaining({ + selection: expect.objectContaining({ stepId: "step-repeat-b", nodeId: "node_1" }), + executions: [ + { stepId: "step-repeat-a", nodeId: "node_1", state: "completed" }, + { stepId: "step-repeat-b", nodeId: "node_1", state: "completed" }, + ], + }), + ); + }); + + it("opens only the exact authorized spend-detail request, including its original timestamp", async () => { + const network = vi + .fn() + .mockResolvedValue( + Response.json({ messages: [{ content: "Stored prompt" }], response: { content: "Stored response" } }), + ); + vi.stubGlobal("fetch", network); + renderWithProviders(); + fireEvent.click(await screen.findByRole("button", { name: "View spend record" })); + expect(await screen.findByLabelText("Spend record fixture-request-summary JSON")).toHaveTextContent( + "Stored response", + ); + expect(network).toHaveBeenCalledTimes(1); + const [url, init] = network.mock.calls[0]; + const parsed = new URL(String(url), "https://gateway.example.test"); + expect(parsed.pathname).toBe("/spend/logs/ui/fixture-request-summary"); + expect(parsed.searchParams.get("start_date")).toBe(fixtureTraces["run-fixture-completed"].steps[0].started_at); + expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer 123"); + }); + + it("shows the failed step and keeps its trace available when the pinned definition fails", async () => { + const source = createWorkflowRunFixtureSource(); + const getDefinition = vi.fn(source.getDefinition).mockRejectedValueOnce(new Error("Missing snapshot")); + renderWithProviders(); + expect(await screen.findByText("ValidationError: summary must be a string. Received null.")).toBeVisible(); + fireEvent.click(await screen.findByRole("button", { name: "Retry definition" })); + await waitFor(() => expect(screen.queryByRole("button", { name: "Retry definition" })).not.toBeInTheDocument()); + expect(getDefinition).toHaveBeenLastCalledWith( + expect.objectContaining({ snapshotId: "snapshot-fixture-v1", runId: "run-fixture-failed" }), + ); + expect(screen.getByLabelText("Output JSON")).toHaveTextContent("null"); + }); + + it("does not display a late response from the previously selected run", async () => { + const source = createWorkflowRunFixtureSource(); + const delayed = Promise.withResolvers(); + const getSteps: WorkflowRunDataSource["getSteps"] = (request) => + request.runId === "run-fixture-running" ? delayed.promise : source.getSteps(request); + const graphState = vi.fn<(state: WorkflowRunGraphState | null) => void>(); + renderWithProviders(); + fireEvent.click(await screen.findByRole("button", { name: "run-fixture-running" })); + expect(screen.getByText("Loading step trace…")).toBeVisible(); + fireEvent.click(screen.getByRole("button", { name: "run-fixture-completed" })); + expect(await screen.findByRole("button", { name: "Inspect step 3 node_3" })).toBeVisible(); + await act(async () => delayed.resolve(fixtureTraces["run-fixture-running"])); + expect(graphState).toHaveBeenLastCalledWith(expect.objectContaining({ runId: "run-fixture-completed" })); + expect(screen.getByRole("region", { name: "Run inspector" })).not.toHaveTextContent("run-fixture-running"); + }); + + it("polls active runs and stops after the gateway reports completion", async () => { + const source = createWorkflowRunFixtureSource(); + const running = fixtureTraces["run-fixture-running"]; + const completed: RunSteps = { + ...running, + state: "completed", + steps: running.steps.map((step) => ({ ...step, state: "completed", finished_at: step.deadline })), + }; + const getSteps = vi.fn(source.getSteps).mockResolvedValueOnce(running).mockResolvedValue(completed); + vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] }); + renderWithProviders(); + expect(await screen.findByText("Updates every 2 seconds")).toBeVisible(); + await act(async () => { + await vi.advanceTimersByTimeAsync(2000); + }); + await waitFor(() => expect(screen.queryByText("Updates every 2 seconds")).not.toBeInTheDocument()); + await act(async () => { + await vi.advanceTimersByTimeAsync(6000); + }); + expect(getSteps).toHaveBeenCalledTimes(2); + }); + + it("shows an empty pending trace without inventing completed steps or cost", async () => { + renderWithProviders(); + expect(await screen.findByText("No steps have been reported yet.")).toBeVisible(); + const inspector = screen.getByRole("region", { name: "Run inspector" }); + expect(within(inspector).getByText("Pending")).toBeVisible(); + expect(within(inspector).getByText("Not reported")).toBeVisible(); + expect(within(inspector).queryByRole("button", { name: "View spend record" })).not.toBeInTheDocument(); + }); + + it("clears a prior session's trace while the replacement source loads", async () => { + const source = createWorkflowRunFixtureSource(); + const graphState = vi.fn<(state: WorkflowRunGraphState | null) => void>(); + const rendered = renderWithProviders( + , + ); + expect(await screen.findByRole("button", { name: "Inspect step 3 node_3" })).toBeVisible(); + const pending = Promise.withResolvers(); + const replacement = { ...source, cacheKey: "another-authenticated-session", getSteps: () => pending.promise }; + rendered.rerender(); + expect(screen.queryByRole("region", { name: "Run inspector" })).not.toBeInTheDocument(); + expect(graphState).toHaveBeenLastCalledWith(null); + await act(async () => pending.resolve(fixtureTraces["run-fixture-completed"])); + expect(await screen.findByRole("region", { name: "Run inspector" })).toBeVisible(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.tsx b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.tsx new file mode 100644 index 00000000000..92487c8e9a7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunDebugger.tsx @@ -0,0 +1,147 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; +import { Button } from "@/components/ui/button"; +import type { + PinnedWorkflowDefinition, + RunSteps, + WorkflowRunDataSource, + WorkflowRunGraphState, + WorkflowStepSelection, +} from "./contracts"; +import { isActive, stepSelection } from "./presentation"; +import { WorkflowRunHistory } from "./WorkflowRunHistory"; +import { WorkflowRunInspector } from "./WorkflowRunInspector"; + +export interface WorkflowRunDebuggerProps { + readonly workflowId: string; + readonly dataSource: WorkflowRunDataSource; + readonly runId: string | null; + readonly onSelectRun: (runId: string) => void; + readonly selectedStepId?: string | null; + readonly onSelectStep?: (selection: WorkflowStepSelection) => void; + readonly onGraphStateChange?: (state: WorkflowRunGraphState | null) => void; + readonly pageSize?: number; +} + +export function WorkflowRunDebugger(props: WorkflowRunDebuggerProps) { + return ( +
+ + +
+ ); +} + +function RunInspection({ + workflowId, + dataSource, + runId, + selectedStepId, + onSelectStep, + onGraphStateChange, +}: WorkflowRunDebuggerProps) { + const [localStepId, setLocalStepId] = useState(null); + const traceOptions: UseQueryOptions = { + queryKey: ["workflowSteps", dataSource.cacheKey, workflowId, runId], + queryFn: ({ signal }) => dataSource.getSteps({ runId: runId!, signal }), + enabled: runId !== null, + retry: false, + refetchInterval: (query) => (query.state.data && isActive(query.state.data.state) ? 2000 : false), + }; + const traceQuery = useQuery(traceOptions); + const trace = traceQuery.data; + const definitionOptions: UseQueryOptions = { + queryKey: ["workflowRunDefinition", dataSource.cacheKey, workflowId, runId, trace?.snapshot_id], + queryFn: ({ signal }) => { + const request = { workflowId, runId: runId!, snapshotId: trace!.snapshot_id, signal }; + return dataSource.getDefinition(request); + }, + enabled: trace !== undefined, + retry: false, + staleTime: Infinity, + }; + const definitionQuery = useQuery(definitionOptions); + const selectionId = selectedStepId === undefined ? localStepId : selectedStepId; + const defaultStep = + trace?.steps.find((step) => step.step_id === selectionId) ?? + trace?.steps.find((step) => step.state === "failed") ?? + trace?.steps[0]; + const selected = + selectedStepId === undefined ? defaultStep : trace?.steps.find((step) => step.step_id === selectedStepId); + const graphState = useMemo( + () => + trace + ? { + runId: trace.run_id, + snapshotId: trace.snapshot_id, + state: trace.state, + definition: definitionQuery.data ?? null, + selection: selected ? stepSelection(trace, selected) : null, + executions: trace.steps.map((step) => ({ stepId: step.step_id, nodeId: step.node_id, state: step.state })), + } + : null, + [trace, definitionQuery.data, selected], + ); + useEffect(() => { + onGraphStateChange?.(graphState); + return () => onGraphStateChange?.(null); + }, [graphState, onGraphStateChange]); + const selectStep = (stepId: string) => { + const step = trace?.steps.find((candidate) => candidate.step_id === stepId); + if (!trace || !step) return; + setLocalStepId(stepId); + onSelectStep?.(stepSelection(trace, step)); + }; + if (runId === null) + return ( +

+ Select a run to explore its steps and pinned definition. +

+ ); + return ( +
+ {traceQuery.isPending && ( +

+ Loading step trace… +

+ )} + {traceQuery.isError && ( +
+ Could not refresh this run. Any displayed steps are from the last successful response. + +
+ )} + {trace && definitionQuery.isPending && ( +

+ Loading this run's pinned definition… +

+ )} + {definitionQuery.isError && ( +
+ Pinned definition unavailable. Step traces are still shown. + +
+ )} + {trace && ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunHistory.tsx b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunHistory.tsx new file mode 100644 index 00000000000..b41caa962d6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunHistory.tsx @@ -0,0 +1,160 @@ +"use client"; + +import { useState } from "react"; +import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; +import { ChevronLeft, ChevronRight, RefreshCw } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { cn } from "@/lib/cva.config"; +import { runStates, type RunStatusFilter, type WorkflowRunDataSource, type WorkflowRunPage } from "./contracts"; +import { formatTimestamp, stateLabels } from "./presentation"; +import { RunStatusBadge } from "./RunStatusBadge"; + +export interface WorkflowRunHistoryProps { + readonly workflowId: string; + readonly dataSource: WorkflowRunDataSource; + readonly selectedRunId: string | null; + readonly onSelectRun: (runId: string) => void; + readonly pageSize?: number; +} + +export function WorkflowRunHistory(props: WorkflowRunHistoryProps) { + return ; +} + +function ScopedHistory({ workflowId, dataSource, selectedRunId, onSelectRun, pageSize = 20 }: WorkflowRunHistoryProps) { + const [status, setStatus] = useState("all"); + const [cursors, setCursors] = useState([null]); + const cursor = cursors[cursors.length - 1]; + const historyOptions: UseQueryOptions = { + queryKey: ["workflowRuns", dataSource.cacheKey, workflowId, status, cursor, pageSize], + queryFn: ({ signal }) => { + const request = { workflowId, cursor, status, limit: pageSize, signal }; + return dataSource.listRuns(request); + }, + retry: false, + refetchInterval: cursor === null ? 5000 : false, + }; + const history = useQuery(historyOptions); + const changeStatus = (value: RunStatusFilter | null) => { + setStatus(value ?? "all"); + setCursors([null]); + }; + const nextCursor = history.data?.next_cursor; + return ( +
+
+
+

Run history

+

Select a run to inspect its execution

+
+
+ + +
+
+ {history.isError && ( +
+ Could not load run history. Check your access and try again. + +
+ )} +
+ + + + Run + Status + Started + Snapshot + + + + {history.data?.runs.map((run) => ( + + + + + + + + + {formatTimestamp(run.created_at)} + + + {run.snapshot_id} + + + ))} + +
+ {history.isPending && ( +

+ Loading runs… +

+ )} + {history.data?.runs.length === 0 && ( +

No runs match this filter.

+ )} +
+
+ Page {cursors.length} +
+ + +
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunInspector.tsx b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunInspector.tsx new file mode 100644 index 00000000000..fbedb4b0acf --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowRunInspector.tsx @@ -0,0 +1,170 @@ +"use client"; + +import { useState } from "react"; +import { ChevronRight, FileText } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/cva.config"; +import type { PinnedWorkflowDefinition, RunSteps, StepTrace } from "./contracts"; +import { formatCost, formatTimestamp, isActive, stepDescription, stepDuration } from "./presentation"; +import { RunStatusBadge } from "./RunStatusBadge"; +import { TracePayload } from "./TracePayload"; +import { WorkflowStepSpendDetails } from "./WorkflowStepSpendDetails"; + +export interface WorkflowRunInspectorProps { + readonly trace: RunSteps; + readonly definition: PinnedWorkflowDefinition | null; + readonly selectedStepId: string | null; + readonly onSelectStep: (stepId: string) => void; +} + +function StepDetails({ step }: { step: StepTrace }) { + const [showSpend, setShowSpend] = useState(false); + const metrics = [ + ["Duration", stepDuration(step)], + ["Attempts", step.attempts.toString()], + ["Input tokens", step.prompt_tokens.toLocaleString()], + ["Output tokens", step.completion_tokens.toLocaleString()], + ["Cost", formatCost(step.response_cost)], + ]; + return ( +
+
+
+

{step.node_id}

+

{step.step_id}

+
+ +
+
+ {metrics.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+
+
+
Started
+
{formatTimestamp(step.started_at)}
+
+
+
Finished
+
{step.finished_at ? formatTimestamp(step.finished_at) : "Not finished"}
+
+
+
Deadline
+
{formatTimestamp(step.deadline)}
+
+
+ {step.error !== null && ( +
+

Step error

+
{step.error}
+
+ )} +
+ + +
+
+ {step.request_id === null ? ( +

No spend-log request ID was reported for this step.

+ ) : ( +
+
+

{step.request_id}

+ +
+ {showSpend && } +
+ )} +
+
+ ); +} + +export function WorkflowRunInspector({ trace, definition, selectedStepId, onSelectStep }: WorkflowRunInspectorProps) { + const selectedStep = trace.steps.find((step) => step.step_id === selectedStepId); + const reportedCosts = trace.steps.flatMap((step) => (step.response_cost === null ? [] : [step.response_cost])); + const reportedCost = reportedCosts.length ? reportedCosts.reduce((total, cost) => total + cost, 0) : null; + const costsIncomplete = trace.steps.some((step) => step.response_cost === null); + return ( +
+
+
+

Run inspector

+
+ {isActive(trace.state) && Updates every 2 seconds} + +
+
+
+
+
Run ID
+
{trace.run_id}
+
+
+
Pinned snapshot
+
{trace.snapshot_id}
+
+
+
Reported step cost
+
+ {formatCost(reportedCost)} + {costsIncomplete && " · Partial"} +
+
+
+ {trace.cancel_requested && ( +

+ Cancellation requested. Waiting for the gateway to report the final state. +

+ )} +
+
+ + {selectedStep ? ( + + ) : ( +

+ {trace.steps.length ? "Select a step to inspect its details." : "No steps have been reported yet."} +

+ )} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/WorkflowStepSpendDetails.tsx b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowStepSpendDetails.tsx new file mode 100644 index 00000000000..8c9b4873073 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/WorkflowStepSpendDetails.tsx @@ -0,0 +1,37 @@ +"use client"; + +import { z } from "zod"; +import { useLogDetails } from "@/app/(dashboard)/hooks/logDetails/useLogDetails"; +import { Button } from "@/components/ui/button"; +import { TracePayload } from "./TracePayload"; + +export interface WorkflowStepSpendDetailsProps { + readonly requestId: string; + readonly startedAt: string; +} + +export function WorkflowStepSpendDetails({ requestId, startedAt }: WorkflowStepSpendDetailsProps) { + const details = useLogDetails(requestId, startedAt, true); + const payload = z.json().safeParse(details.data); + if (details.isError) { + return ( +
+

+ Spend record unavailable. It may not be stored yet, may have expired, or your account may not have access. +

+ +
+ ); + } + if (details.isPending || details.isFetching) + return ( +

+ Loading authorized spend record… +

+ ); + if (!payload.success || payload.data === null) + return

No stored details are available for this request.

; + return ; +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/api.test.ts b/ui/litellm-dashboard/src/components/workflows/runs/api.test.ts new file mode 100644 index 00000000000..87baa98cc3a --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/api.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it, vi } from "vitest"; +import { createApiClient } from "@/lib/http/client"; +import { createWorkflowRunApi } from "./api"; +import { runStepsSchema } from "./contracts"; +import { + fixtureDefinitions, + fixtureRunSummaries, + fixtureTraces, + fixtureWorkflowId, +} from "../../../../tests/fixtures/workflow-runs/data"; +import { formatCost, stepDuration } from "./presentation"; + +function setup(body: unknown) { + const transport = vi.fn().mockImplementation(async () => Response.json(body)); + const client = createApiClient({ getBaseUrl: () => "https://gateway.example.test", fetchImpl: transport }); + const api = createWorkflowRunApi({ accessToken: "test-token", teamId: "test-team", client }); + const signal = new AbortController().signal; + return { transport, api, signal }; +} + +describe("workflow run API boundary", () => { + it("scopes and paginates the run list without querying spend logs", async () => { + const run = fixtureRunSummaries[0]; + const { api, transport, signal } = setup({ runs: [{ ...run, status: run.state }], next_cursor: "opaque-next" }); + const request = { + workflowId: fixtureWorkflowId, + cursor: "opaque-current", + status: "completed" as const, + limit: 10, + signal, + }; + const result = await api.listRuns(request); + expect(result).toEqual({ runs: [run], next_cursor: "opaque-next" }); + const [url, init] = transport.mock.calls[0]; + const parsed = new URL(String(url)); + expect(parsed.pathname).toBe("/v1/workflows/runs"); + const expectedQuery = { + team_id: "test-team", + workflow_id: fixtureWorkflowId, + after: "opaque-current", + status: "completed", + limit: "10", + }; + expect(Object.fromEntries(parsed.searchParams)).toEqual(expectedQuery); + expect(init?.signal).toBe(signal); + expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer test-token"); + expect(transport).toHaveBeenCalledTimes(1); + }); + + it("rejects cross-workflow rows", async () => { + const { api, signal } = setup({ + runs: [{ ...fixtureRunSummaries[0], workflow_id: "other", status: "completed" }], + next_cursor: null, + }); + const request = { workflowId: fixtureWorkflowId, cursor: null, status: "all" as const, limit: 20, signal }; + await expect(api.listRuns(request)).rejects.toThrow("different workflow"); + }); + + it("retrieves the exact run trace and rejects a different run", async () => { + const trace = fixtureTraces["run-fixture-completed"]; + const { api, transport, signal } = setup(trace); + await expect(api.getSteps({ runId: trace.run_id, signal })).resolves.toEqual(trace); + expect(String(transport.mock.calls[0][0])).toContain(`/runs/${trace.run_id}/steps`); + await expect(api.getSteps({ runId: "another-run", signal })).rejects.toThrow("different run"); + }); + + it("rejects a draft or a different workflow in place of the pinned snapshot", async () => { + const pinned = fixtureDefinitions["snapshot-fixture-v1"]; + const { api, transport, signal } = setup(pinned); + const request = { + runId: "run-fixture-completed", + workflowId: fixtureWorkflowId, + snapshotId: pinned.snapshot_id, + signal, + }; + await expect(api.getDefinition(request)).resolves.toEqual(pinned); + expect(String(transport.mock.calls[0][0])).toContain(`/runs/${request.runId}/definition`); + await expect(api.getDefinition({ ...request, snapshotId: "current-draft" })).rejects.toThrow("pinned snapshot"); + await expect(api.getDefinition({ ...request, workflowId: "other" })).rejects.toThrow("pinned snapshot"); + }); + + it("propagates cancellation to the transport", async () => { + const controller = new AbortController(); + const transport = vi.fn().mockImplementation(async (_url, init) => { + init?.signal?.throwIfAborted(); + return Response.json(fixtureTraces["run-fixture-completed"]); + }); + const client = createApiClient({ getBaseUrl: () => "https://gateway.example.test", fetchImpl: transport }); + const api = createWorkflowRunApi({ accessToken: "test", teamId: "test", client }); + controller.abort(); + await expect(api.getSteps({ runId: "run-fixture-completed", signal: controller.signal })).rejects.toThrow(); + }); +}); + +describe("trace identity and values", () => { + it("keeps distinct steps for repeated nodes and rejects duplicate step IDs", () => { + const trace = fixtureTraces["run-fixture-repeated-node"]; + expect(runStepsSchema.parse(trace).steps.map((step) => step.step_id)).toEqual(["step-repeat-a", "step-repeat-b"]); + expect(runStepsSchema.safeParse({ ...trace, steps: [trace.steps[0], trace.steps[0]] }).success).toBe(false); + const foreignStep = { ...trace.steps[0], run_id: "other" }; + expect(runStepsSchema.safeParse({ ...trace, steps: [foreignStep] }).success).toBe(false); + }); + + it("preserves scalar JSON, null request IDs and recorded zero cost", () => { + const step = runStepsSchema.parse(fixtureTraces["run-fixture-completed"]).steps[1]; + expect(step.input).toBe(false); + expect(step.output).toBe(0); + expect(step.request_id).toBeNull(); + expect(formatCost(step.response_cost)).toBe("$0.000000"); + expect(formatCost(null)).toBe("Not reported"); + }); + + it("reports real elapsed time and distinguishes active and unreported duration", () => { + const step = fixtureTraces["run-fixture-completed"].steps[1]; + expect(stepDuration(step)).toBe("25 ms"); + expect(stepDuration({ ...step, state: "running", finished_at: null })).toBe("In progress"); + expect(stepDuration({ ...step, state: "failed", finished_at: null })).toBe("Not reported"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/workflows/runs/api.ts b/ui/litellm-dashboard/src/components/workflows/runs/api.ts new file mode 100644 index 00000000000..e67d7ba5f95 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/api.ts @@ -0,0 +1,82 @@ +import { z } from "zod"; +import { apiClient } from "@/components/networking"; +import type { ApiClient } from "@/lib/http/client"; +import { runStateSchema, runStepsSchema, type WorkflowRunDataSource } from "./contracts"; + +const runSummaryFields = { + run_id: z.string().min(1), + workflow_id: z.string().min(1), + snapshot_id: z.string().min(1), + status: runStateSchema, + created_at: z.iso.datetime({ offset: true }), +}; +const runListSchema = z.object({ + runs: z.array(z.object(runSummaryFields)), + next_cursor: z.string().min(1).nullable(), +}); + +const pinnedDefinitionSchema = z.object({ + workflow_id: z.string().min(1), + snapshot_id: z.string().min(1), + bundle: z.record(z.string(), z.json()), +}); + +export interface WorkflowRunApiOptions { + readonly accessToken: string; + readonly teamId: string; + readonly client?: Pick; +} + +export function createWorkflowRunApi({ + accessToken, + teamId, + client = apiClient, +}: WorkflowRunApiOptions): WorkflowRunDataSource { + return { + cacheKey: crypto.randomUUID(), + async listRuns({ workflowId, cursor, status, limit, signal }) { + const page = runListSchema.parse( + await client.get("/v1/workflows/runs", { + accessToken, + signal, + query: { + team_id: teamId, + workflow_id: workflowId, + after: cursor, + status: status === "all" ? undefined : status, + limit, + }, + }), + ); + if (page.runs.some((run) => run.workflow_id !== workflowId)) { + throw new Error("The run list returned a different workflow"); + } + return { + runs: page.runs.map(({ status: state, ...run }) => ({ ...run, state })), + next_cursor: page.next_cursor, + }; + }, + async getSteps({ runId, signal }) { + const trace = runStepsSchema.parse( + await client.get(`/v1/workflows/runs/${encodeURIComponent(runId)}/steps`, { + accessToken, + signal, + }), + ); + if (trace.run_id !== runId) throw new Error("The trace returned a different run"); + return trace; + }, + async getDefinition({ runId, workflowId, snapshotId, signal }) { + const definition = pinnedDefinitionSchema.parse( + await client.get(`/v1/workflows/runs/${encodeURIComponent(runId)}/definition`, { + accessToken, + signal, + }), + ); + if (definition.snapshot_id !== snapshotId || definition.workflow_id !== workflowId) { + throw new Error("The definition does not match this run's pinned snapshot"); + } + return definition; + }, + }; +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/contracts.ts b/ui/litellm-dashboard/src/components/workflows/runs/contracts.ts new file mode 100644 index 00000000000..7758be88084 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/contracts.ts @@ -0,0 +1,107 @@ +import { z } from "zod"; + +export const runStates = ["pending", "running", "completed", "failed", "cancelled"] as const; +export const runStateSchema = z.enum(runStates); +const timestamp = z.iso.datetime({ offset: true }); +const identifier = z.string().min(1); + +const stepTraceFields = { + step_id: identifier, + run_id: identifier, + node_id: identifier, + state: runStateSchema, + input: z.json(), + output: z.json(), + attempts: z.number().int().nonnegative(), + deadline: timestamp, + started_at: timestamp, + finished_at: timestamp.nullable(), + error: z.string().nullable(), + prompt_tokens: z.number().int().nonnegative(), + completion_tokens: z.number().int().nonnegative(), + response_cost: z.number().finite().nonnegative().nullable(), + request_id: identifier.nullable(), +}; +export const stepTraceSchema = z.object(stepTraceFields); + +const runStepsFields = { + run_id: identifier, + snapshot_id: identifier, + state: runStateSchema, + cancel_requested: z.boolean(), + steps: z.array(stepTraceSchema), +}; +export const runStepsSchema = z + .object(runStepsFields) + .refine((run) => run.steps.every((step) => step.run_id === run.run_id), "Step belongs to another run") + .refine((run) => new Set(run.steps.map((step) => step.step_id)).size === run.steps.length, "Duplicate step identity"); + +export type RunState = z.infer; +export type StepTrace = z.infer; +export type RunSteps = z.infer; +export type JsonValue = z.infer>; +export type RunStatusFilter = RunState | "all"; + +export interface WorkflowRunSummary { + readonly run_id: string; + readonly workflow_id: string; + readonly snapshot_id: string; + readonly state: RunState; + readonly created_at: string; +} + +export interface WorkflowRunPage { + readonly runs: readonly WorkflowRunSummary[]; + readonly next_cursor: string | null; +} + +export interface PinnedWorkflowDefinition { + readonly workflow_id: string; + readonly snapshot_id: string; + readonly bundle: Readonly>; +} + +export interface ListWorkflowRuns { + readonly workflowId: string; + readonly cursor: string | null; + readonly status: RunStatusFilter; + readonly limit: number; + readonly signal: AbortSignal; +} + +export interface ReadRun { + readonly runId: string; + readonly signal: AbortSignal; +} + +export interface ReadPinnedDefinition extends ReadRun { + readonly workflowId: string; + readonly snapshotId: string; +} + +export interface WorkflowRunDataSource { + readonly cacheKey: string; + listRuns: (request: ListWorkflowRuns) => Promise; + getSteps: (request: ReadRun) => Promise; + getDefinition: (request: ReadPinnedDefinition) => Promise; +} + +export interface WorkflowStepSelection { + readonly runId: string; + readonly snapshotId: string; + readonly stepId: string; + readonly nodeId: string; +} + +export interface WorkflowRunGraphState { + readonly runId: string; + readonly snapshotId: string; + readonly state: RunState; + readonly definition: PinnedWorkflowDefinition | null; + readonly selection: WorkflowStepSelection | null; + readonly executions: readonly { + readonly stepId: string; + readonly nodeId: string; + readonly state: RunState; + }[]; +} diff --git a/ui/litellm-dashboard/src/components/workflows/runs/index.ts b/ui/litellm-dashboard/src/components/workflows/runs/index.ts new file mode 100644 index 00000000000..028e8032c97 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/index.ts @@ -0,0 +1,20 @@ +export { createWorkflowRunApi, type WorkflowRunApiOptions } from "./api"; +export { WorkflowRunHistory, type WorkflowRunHistoryProps } from "./WorkflowRunHistory"; +export { WorkflowRunInspector, type WorkflowRunInspectorProps } from "./WorkflowRunInspector"; +export { WorkflowRunDebugger, type WorkflowRunDebuggerProps } from "./WorkflowRunDebugger"; +export type { + JsonValue, + ListWorkflowRuns, + PinnedWorkflowDefinition, + ReadPinnedDefinition, + ReadRun, + RunState, + RunStatusFilter, + RunSteps, + StepTrace, + WorkflowRunDataSource, + WorkflowRunGraphState, + WorkflowRunPage, + WorkflowRunSummary, + WorkflowStepSelection, +} from "./contracts"; diff --git a/ui/litellm-dashboard/src/components/workflows/runs/presentation.ts b/ui/litellm-dashboard/src/components/workflows/runs/presentation.ts new file mode 100644 index 00000000000..6ac3a879181 --- /dev/null +++ b/ui/litellm-dashboard/src/components/workflows/runs/presentation.ts @@ -0,0 +1,54 @@ +import type { PinnedWorkflowDefinition, RunState, RunSteps, StepTrace, WorkflowStepSelection } from "./contracts"; + +export const stateLabels: Record = { + pending: "Pending", + running: "Running", + completed: "Completed", + failed: "Failed", + cancelled: "Cancelled", +}; + +export function isActive(state: RunState): boolean { + return state === "pending" || state === "running"; +} + +export function stepDuration(step: StepTrace): string { + if (step.finished_at === null) return isActive(step.state) ? "In progress" : "Not reported"; + const milliseconds = Date.parse(step.finished_at) - Date.parse(step.started_at); + if (milliseconds < 0 || !Number.isFinite(milliseconds)) return "Not reported"; + return milliseconds < 1000 ? `${milliseconds} ms` : `${(milliseconds / 1000).toFixed(2)} s`; +} + +export function formatCost(cost: number | null): string { + return cost === null ? "Not reported" : `$${cost.toFixed(6)}`; +} + +export function formatTimestamp(timestamp: string): string { + const options: Intl.DateTimeFormatOptions = { + year: "numeric", + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + timeZoneName: "short", + }; + return new Date(timestamp).toLocaleString(undefined, options); +} + +export function stepSelection(trace: RunSteps, step: StepTrace): WorkflowStepSelection { + return { runId: trace.run_id, snapshotId: trace.snapshot_id, stepId: step.step_id, nodeId: step.node_id }; +} + +export function stepDescription(definition: PinnedWorkflowDefinition | null, nodeId: string): string { + const nodes = definition?.bundle.nodes; + if (!Array.isArray(nodes)) return "Workflow step"; + const node = nodes.find((entry) => { + if (entry === null || typeof entry !== "object" || Array.isArray(entry)) return false; + return entry.id === nodeId; + }); + if (node === null || typeof node !== "object" || Array.isArray(node)) return "Workflow step"; + if (node.type === "python") return "Python"; + if (node.type === "chat_completion") return typeof node.model === "string" ? node.model : "Chat completion"; + return "Workflow step"; +} diff --git a/ui/litellm-dashboard/tests/fixtures/workflow-runs/WorkflowRunDemo.tsx b/ui/litellm-dashboard/tests/fixtures/workflow-runs/WorkflowRunDemo.tsx new file mode 100644 index 00000000000..23ce13b7bfc --- /dev/null +++ b/ui/litellm-dashboard/tests/fixtures/workflow-runs/WorkflowRunDemo.tsx @@ -0,0 +1,44 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { createWorkflowRunApi, WorkflowRunDebugger, type WorkflowRunGraphState } from "@/components/workflows/runs"; +import { useAuth } from "@/contexts/AuthContext"; +import { fixtureWorkflowId } from "./data"; + +export default function WorkflowRunDemo() { + const { accessToken } = useAuth(); + const source = useMemo( + () => createWorkflowRunApi({ accessToken: accessToken ?? "", teamId: "fixture-team" }), + [accessToken], + ); + const [runId, setRunId] = useState(null); + const [graphState, setGraphState] = useState(null); + return ( +
+
+

Workflows / Execution history

+

Document summary

+

+ Inspect runs, follow each step, and debug its exact inputs and outputs +

+
+
+ Fixture preview · Synthetic traces and spend records. No workflow execution or live billing. +
+ + {graphState && ( +

+ Graph selection · {graphState.selection?.nodeId ?? "None"} ·{" "} + {graphState.selection?.stepId ?? "No step selected"} · {graphState.snapshotId} +

+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/tests/fixtures/workflow-runs/data.ts b/ui/litellm-dashboard/tests/fixtures/workflow-runs/data.ts new file mode 100644 index 00000000000..ffd7348b9ed --- /dev/null +++ b/ui/litellm-dashboard/tests/fixtures/workflow-runs/data.ts @@ -0,0 +1,225 @@ +import type { + PinnedWorkflowDefinition, + RunSteps, + StepTrace, + WorkflowRunDataSource, + WorkflowRunSummary, +} from "@/components/workflows/runs/contracts"; + +export const fixtureWorkflowId = "fixture-summary-workflow"; + +const nodeDefaults = { timeout_seconds: 30, retry: { max_attempts: 2, backoff_seconds: 1 } }; +export const fixtureDefinitions: Readonly> = { + "snapshot-fixture-v1": { + workflow_id: fixtureWorkflowId, + snapshot_id: "snapshot-fixture-v1", + bundle: { + contract_version: "1", + name: "document-summary", + timeout_seconds: 90, + input_schema: { type: "array", items: { type: "object" } }, + output: { from: "node_3", output: "result" }, + nodes: [ + { + ...nodeDefaults, + id: "node_1", + type: "chat_completion", + model: "summary-model", + messages: { from: "$input", output: "messages" }, + }, + { + ...nodeDefaults, + id: "node_2", + type: "python", + inputs: { summary: { from: "node_1", output: "result" } }, + program: { + contract_version: "1", + function_name: "format_summary", + parameter_names: ["summary"], + source: "def format_summary(summary: str) -> str:\n return summary.strip()\n", + input_schema: { + type: "object", + properties: { summary: { type: "string" } }, + required: ["summary"], + additionalProperties: false, + }, + output_schema: { type: "string" }, + source_sha256: "a7d6ef8657708891bfaa6fb481ef66048bf030877a0197b94e6e0c28859c98573", + }, + }, + { + ...nodeDefaults, + id: "node_3", + type: "chat_completion", + model: "review-model", + messages: { from: "node_2", output: "result" }, + }, + ], + }, + }, +}; + +const startedAt = "2026-11-06T08:30:00Z"; +const stepDefaults = { + state: "completed", + input: { messages: [{ role: "user", content: "Summarize the release notes in one sentence." }] }, + output: { summary: "The release improves workflow debugging.", confidence: 0.94 }, + attempts: 1, + deadline: "2026-11-06T08:31:00Z", + started_at: startedAt, + finished_at: "2026-11-06T08:30:02Z", + error: null, + prompt_tokens: 142, + completion_tokens: 36, + response_cost: 0.00042, + request_id: "fixture-request-summary", +} satisfies Omit; + +export const fixtureTraces: Readonly> = { + "run-fixture-completed": { + run_id: "run-fixture-completed", + snapshot_id: "snapshot-fixture-v1", + state: "completed", + cancel_requested: false, + steps: [ + { ...stepDefaults, run_id: "run-fixture-completed", step_id: "step-completed-1", node_id: "node_1" }, + { + ...stepDefaults, + run_id: "run-fixture-completed", + step_id: "step-completed-2", + node_id: "node_2", + input: false, + output: 0, + prompt_tokens: 0, + completion_tokens: 0, + response_cost: 0, + request_id: null, + started_at: "2026-11-06T08:30:02Z", + finished_at: "2026-11-06T08:30:02.025Z", + }, + { + ...stepDefaults, + run_id: "run-fixture-completed", + step_id: "step-completed-3", + node_id: "node_3", + input: { accepted: true, summary: "The release improves workflow debugging." }, + output: "Ready to publish.", + request_id: "fixture-request-review", + started_at: "2026-11-06T08:30:02.025Z", + finished_at: "2026-11-06T08:30:03.150Z", + }, + ], + }, + "run-fixture-running": { + run_id: "run-fixture-running", + snapshot_id: "snapshot-fixture-v1", + state: "running", + cancel_requested: false, + steps: [ + { ...stepDefaults, run_id: "run-fixture-running", step_id: "step-running-1", node_id: "node_1" }, + { + ...stepDefaults, + run_id: "run-fixture-running", + step_id: "step-running-2", + node_id: "node_2", + state: "running", + input: "", + output: null, + prompt_tokens: 0, + completion_tokens: 0, + response_cost: null, + request_id: null, + finished_at: null, + }, + ], + }, + "run-fixture-failed": { + run_id: "run-fixture-failed", + snapshot_id: "snapshot-fixture-v1", + state: "failed", + cancel_requested: false, + steps: [ + { ...stepDefaults, run_id: "run-fixture-failed", step_id: "step-failed-1", node_id: "node_1" }, + { + ...stepDefaults, + run_id: "run-fixture-failed", + step_id: "step-failed-2", + node_id: "node_2", + state: "failed", + input: { summary: null }, + output: null, + attempts: 2, + error: "ValidationError: summary must be a string. Received null.", + prompt_tokens: 0, + completion_tokens: 0, + response_cost: null, + request_id: null, + }, + ], + }, + "run-fixture-repeated-node": { + run_id: "run-fixture-repeated-node", + snapshot_id: "snapshot-fixture-v1", + state: "completed", + cancel_requested: false, + steps: [ + { + ...stepDefaults, + run_id: "run-fixture-repeated-node", + step_id: "step-repeat-a", + node_id: "node_1", + output: "First execution", + }, + { + ...stepDefaults, + run_id: "run-fixture-repeated-node", + step_id: "step-repeat-b", + node_id: "node_1", + output: "Second execution", + request_id: "fixture-request-second", + }, + ], + }, + "run-fixture-pending": { + run_id: "run-fixture-pending", + snapshot_id: "snapshot-fixture-v1", + state: "pending", + cancel_requested: false, + steps: [], + }, +}; + +export const fixtureRunSummaries: readonly WorkflowRunSummary[] = Object.values(fixtureTraces).map((trace) => ({ + run_id: trace.run_id, + workflow_id: fixtureWorkflowId, + snapshot_id: trace.snapshot_id, + state: trace.state, + created_at: startedAt, +})); + +export function createWorkflowRunFixtureSource(): WorkflowRunDataSource { + return { + cacheKey: crypto.randomUUID(), + async listRuns({ workflowId, status, cursor, limit, signal }) { + signal.throwIfAborted(); + const matching = fixtureRunSummaries.filter( + (run) => run.workflow_id === workflowId && (status === "all" || run.state === status), + ); + const start = cursor ? matching.findIndex((run) => run.run_id === cursor) + 1 : 0; + const runs = matching.slice(start, start + limit); + return { runs, next_cursor: start + limit < matching.length ? runs[runs.length - 1].run_id : null }; + }, + async getSteps({ runId, signal }) { + signal.throwIfAborted(); + const trace = fixtureTraces[runId]; + if (!trace) throw new Error("Fixture run not found"); + return trace; + }, + async getDefinition({ workflowId, snapshotId, signal }) { + signal.throwIfAborted(); + const definition = fixtureDefinitions[snapshotId]; + if (!definition || definition.workflow_id !== workflowId) throw new Error("Fixture snapshot not found"); + return definition; + }, + }; +} diff --git a/ui/litellm-dashboard/tests/fixtures/workflow-runs/serve-shell.mjs b/ui/litellm-dashboard/tests/fixtures/workflow-runs/serve-shell.mjs new file mode 100644 index 00000000000..76d1f47bf50 --- /dev/null +++ b/ui/litellm-dashboard/tests/fixtures/workflow-runs/serve-shell.mjs @@ -0,0 +1,118 @@ +import { createServer } from "node:http"; +import { readFileSync } from "node:fs"; +import { fixtureRunSummaries, fixtureTraces, fixtureDefinitions } from "./data.ts"; + +const hostname = "127.0.0.1"; +const port = 3378; +const dashboard = "http://workflow-runs.localhost:3377"; +const user = { user_id: "fixture-user", user_email: "preview@example.test", user_role: "proxy_admin" }; +const claims = { + ...user, + key: "fixture-only-not-a-gateway-key", + login_method: "sso", + exp: Math.floor(Date.now() / 1000) + 86400, +}; +const token = `${Buffer.from('{"alg":"HS256","typ":"JWT"}').toString("base64url")}.${Buffer.from(JSON.stringify(claims)).toString("base64url")}.fixture`; +const responses = { + "/litellm/.well-known/litellm-ui-config": { server_root_path: "", proxy_base_url: dashboard }, + "/ui/litellm-runtime-config.json": {}, + "/litellm-runtime-config.json": {}, + "/management/v1/users/me": user, + "/get/ui_theme_settings": { values: {} }, + "/get/ui_settings": { values: { enable_projects_ui: false } }, + "/sso/get/ui_settings": { values: { enable_projects_ui: false } }, + "/api/plugins": [], + "/user/info": { user_id: user.user_id, user_info: { ...user, teams: [], models: [] }, teams: [], keys: [] }, + "/organization/info": [], + "/team/list": [], + "/health/readiness": { status: "healthy" }, + "/health/liveliness": { status: "healthy" }, + "/health/readiness/details": { status: "healthy" }, + "/health/deployment_warnings": { warnings: [], ignore_available: false }, + "/get/user_banner": { enabled: false, message: "", severity: "info", revision: "fixture" }, + "/user/available_users": [], + "/telemetry/ui_events/enabled": { enabled: false }, + "/telemetry/settings": { + groups: [], + destination: "local_table", + environment_variables: [], + set_by_environment: false, + vetoed: false, + }, +}; + +createServer((request, response) => { + const url = new URL(request.url, dashboard); + const pathname = url.pathname; + const send = (payload) => { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(payload)); + }; + if (pathname === "/__workflow_run_fixture_login") { + response.writeHead(302, { + "Set-Cookie": `token=${token}; Path=/; SameSite=Lax; Max-Age=86400`, + Location: `${dashboard}/ui/workflows/run-preview`, + }); + response.end(); + return; + } + if (pathname === "/get_image") { + response.writeHead(200, { "Content-Type": "image/png" }); + response.end(readFileSync(new URL("../../../../../litellm/proxy/logo.png", import.meta.url))); + return; + } + if (pathname.startsWith("/v1/workflows/runs")) { + if (request.headers.authorization !== `Bearer ${claims.key}`) { + response.writeHead(403, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ error: "Fixture authentication required" })); + return; + } + if (pathname === "/v1/workflows/runs") { + const status = url.searchParams.get("status"); + const runs = fixtureRunSummaries.filter( + (run) => run.workflow_id === url.searchParams.get("workflow_id") && (!status || run.state === status), + ); + const after = url.searchParams.get("after"); + const start = after ? runs.findIndex((run) => run.run_id === after) + 1 : 0; + const limit = Number(url.searchParams.get("limit") ?? 20); + const page = runs.slice(start, start + limit); + send({ + runs: page.map(({ state, ...run }) => ({ ...run, status: state })), + next_cursor: start + limit < runs.length ? page.at(-1).run_id : null, + }); + return; + } + const match = pathname.match(/^\/v1\/workflows\/runs\/([^/]+)\/(steps|definition)$/); + const trace = match && fixtureTraces[decodeURIComponent(match[1])]; + if (trace) { + send(match[2] === "steps" ? trace : fixtureDefinitions[trace.snapshot_id]); + return; + } + } + if (pathname.startsWith("/spend/logs/ui/fixture-request-")) { + if (request.headers.authorization !== `Bearer ${claims.key}`) { + response.writeHead(403, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ error: "Fixture authentication required" })); + return; + } + const payload = { + fixture: true, + request_id: pathname.slice("/spend/logs/ui/".length), + messages: [{ role: "user", content: "Summarize the release notes in one sentence." }], + response: { content: "The release improves workflow debugging." }, + }; + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(payload)); + return; + } + if (Object.hasOwn(responses, pathname)) { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(responses[pathname])); + return; + } + process.stdout.write(`Unconfigured shell fixture ${request.method} ${pathname}\n`); + response.writeHead(404, { "Content-Type": "application/json" }); + response.end(JSON.stringify({ error: "No fixture for this request" })); +}).listen(port, hostname, () => { + process.stdout.write(`Fixture dashboard login ${dashboard}/__workflow_run_fixture_login\n`); +});