scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-13 23:14:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Merge main into PR 212

Browse source
This commit is contained in:
Bryan Helmkamp 2026-05-05 09:05:24 -04:00
commit 304927e3a6
No known key found for this signature in database
46 changed files with 3383 additions and 643 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
Cargo.lock generated
View file

@ -1536,7 +1536,7 @@ dependencies = [
[[package]]
name = "fabro-agent"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1575,7 +1575,7 @@ dependencies = [
[[package]]
name = "fabro-api"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-config",
@ -1596,7 +1596,7 @@ dependencies = [
[[package]]
name = "fabro-auth"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1620,7 +1620,7 @@ dependencies = [
[[package]]
name = "fabro-checkpoint"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-config",
@ -1636,7 +1636,7 @@ dependencies = [
[[package]]
name = "fabro-cli"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -1732,7 +1732,7 @@ dependencies = [
[[package]]
name = "fabro-client"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"bytes",
@ -1761,7 +1761,7 @@ dependencies = [
[[package]]
name = "fabro-config"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"chrono",
@ -1788,7 +1788,7 @@ dependencies = [
[[package]]
name = "fabro-core"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"fabro-types",
@ -1803,7 +1803,7 @@ dependencies = [
[[package]]
name = "fabro-dev"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -1823,7 +1823,7 @@ dependencies = [
[[package]]
name = "fabro-devcontainer"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-http",
"fabro-static",
@ -1840,7 +1840,7 @@ dependencies = [
[[package]]
name = "fabro-dump"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"bytes",
@ -1854,7 +1854,7 @@ dependencies = [
[[package]]
name = "fabro-github"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -1876,7 +1876,7 @@ dependencies = [
[[package]]
name = "fabro-graphviz"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-types",
@ -1890,7 +1890,7 @@ dependencies = [
[[package]]
name = "fabro-hooks"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"fabro-agent",
@ -1914,7 +1914,7 @@ dependencies = [
[[package]]
name = "fabro-http"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-static",
"http",
@ -1924,7 +1924,7 @@ dependencies = [
[[package]]
name = "fabro-install"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -1939,7 +1939,7 @@ dependencies = [
[[package]]
name = "fabro-interview"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"dialoguer",
@ -1954,7 +1954,7 @@ dependencies = [
[[package]]
name = "fabro-llm"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1986,7 +1986,7 @@ dependencies = [
[[package]]
name = "fabro-macros"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"clap",
"fabro-options-metadata",
@ -1997,7 +1997,7 @@ dependencies = [
[[package]]
name = "fabro-mcp"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-config",
@ -2013,7 +2013,7 @@ dependencies = [
[[package]]
name = "fabro-model"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-static",
"insta",
@ -2024,7 +2024,7 @@ dependencies = [
[[package]]
name = "fabro-oauth"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"axum",
@ -2046,7 +2046,7 @@ dependencies = [
[[package]]
name = "fabro-options-metadata"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"serde",
"serde_json",
@ -2054,7 +2054,7 @@ dependencies = [
[[package]]
name = "fabro-proc"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"cc",
"libc",
@ -2063,7 +2063,7 @@ dependencies = [
[[package]]
name = "fabro-redact"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"aho-corasick",
"ref-cast",
@ -2079,7 +2079,7 @@ dependencies = [
[[package]]
name = "fabro-retro"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"chrono",
@ -2098,7 +2098,7 @@ dependencies = [
[[package]]
name = "fabro-sandbox"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2138,7 +2138,7 @@ dependencies = [
[[package]]
name = "fabro-server"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2219,7 +2219,7 @@ dependencies = [
[[package]]
name = "fabro-slack"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-http",
"fabro-interview",
@ -2240,18 +2240,18 @@ dependencies = [
[[package]]
name = "fabro-spa"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"rust-embed",
]
[[package]]
name = "fabro-static"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
[[package]]
name = "fabro-store"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"bytes",
@ -2278,7 +2278,7 @@ dependencies = [
[[package]]
name = "fabro-telemetry"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -2304,7 +2304,7 @@ dependencies = [
[[package]]
name = "fabro-template"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-util",
@ -2316,7 +2316,7 @@ dependencies = [
[[package]]
name = "fabro-test"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"assert_cmd",
"axum",
@ -2339,7 +2339,7 @@ dependencies = [
[[package]]
name = "fabro-tracker"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2353,7 +2353,7 @@ dependencies = [
[[package]]
name = "fabro-types"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"clap",
@ -2374,7 +2374,7 @@ dependencies = [
[[package]]
name = "fabro-util"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"console 0.15.11",
@ -2394,7 +2394,7 @@ dependencies = [
[[package]]
name = "fabro-validate"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-graphviz",
"fabro-model",
@ -2404,7 +2404,7 @@ dependencies = [
[[package]]
name = "fabro-vault"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-types",
@ -2416,7 +2416,7 @@ dependencies = [
[[package]]
name = "fabro-workflow"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -7120,7 +7120,7 @@ dependencies = [
[[package]]
name = "twin-github"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"axum",
"base64",
@ -7139,7 +7139,7 @@ dependencies = [
[[package]]
name = "twin-openai"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-stream",

2
Cargo.toml
View file

@ -5,7 +5,7 @@ resolver = "2"
[workspace.package]
edition = "2021"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
license = "MIT"
[workspace.dependencies]

7
apps/fabro-web/app/components/stage-sidebar.tsx
View file

@ -11,14 +11,15 @@ import {
} from "@heroicons/react/24/solid";
import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline";
import { formatDurationSecs } from "../lib/format";
import { ACTIVE_STAGE_STATES } from "../lib/stage-sidebar";
import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar";
export interface Stage {
id: string;
name: string;
status: StageState;
duration: string;
dotId?: string;
nodeId: string;
visit: number;
}
export const statusConfig: Record<StageState, { icon: ComponentType<{ className?: string }>; color: string }> = {
@ -100,7 +101,7 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta
}`}
>
<Icon className={`size-4 shrink-0 ${config.color} ${ACTIVE_STAGE_STATES.has(stage.status) ? "animate-spin" : ""}`} />
<span className="flex-1 truncate">{stage.name}</span>
<span className="flex-1 truncate">{formatStageLabel(stage)}</span>
<span className="font-mono text-xs tabular-nums text-fg-muted">{stageDuration(stage)}</span>
</Link>
</li>

65
apps/fabro-web/app/lib/run-events.test.tsx
View file

@ -49,6 +49,14 @@ describe("queryKeysForRunEvent", () => {
queryKeys.runs.graph("run-1", "TB"),
]);
});
test("stage.retrying invalidates the same keys as other stage events", () => {
const keys = queryKeysForRunEvent("run-1", "stage.retrying", "verify@2");
expect(keys).toContain(queryKeys.runs.stages("run-1"));
expect(keys).toContain(queryKeys.runs.events("run-1", 1000));
expect(keys).toContain(queryKeys.runs.detail("run-1"));
expect(keys).toContain(queryKeys.runs.stageEvents("run-1", "verify@2"));
});
});
describe("subscribeToRunEvents", () => {
@ -172,6 +180,63 @@ describe("subscribeToRunEvents", () => {
coordinator.close();
});
test("envelope with suffixed stage_id invalidates stageEvents(runId, stageId)", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-stage",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
source.emit({
event: "stage.retrying",
run_id: "run-stage",
stage_id: "verify@2",
node_id: "verify",
});
expect(keys).toContain(queryKeys.runs.stageEvents("run-stage", "verify@2"));
expect(keys).toContain(queryKeys.runs.stages("run-stage"));
expect(keys).toContain(queryKeys.runs.events("run-stage", 1000));
expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR"));
expect(keys).toContain(queryKeys.runs.detail("run-stage"));
expect(keys).not.toContain(queryKeys.runs.stageEvents("run-stage", "verify"));
cleanup();
coordinator.close();
});
test("falls back to node_id when an event has no stage_id", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-stage-node",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" });
expect(keys).toContain(queryKeys.runs.stageEvents("run-stage-node", "verify"));
expect(keys).toContain(queryKeys.runs.stages("run-stage-node"));
cleanup();
coordinator.close();
});
test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => {
const firstSource = new FakeEventSource();
const secondSource = new FakeEventSource();

11
apps/fabro-web/app/lib/run-events.ts
View file

@ -19,6 +19,7 @@ interface RunEventPayload extends EventPayload {
event?: string;
run_id?: string;
node_id?: string;
stage_id?: string;
properties?: Record<string, unknown>;
}
@ -42,7 +43,12 @@ const RUN_SUMMARY_EVENTS = new Set([
"run.archived",
"run.unarchived",
]);
const STAGE_EVENTS = new Set(["stage.started", "stage.completed", "stage.failed"]);
const STAGE_EVENTS = new Set([
"stage.started",
"stage.completed",
"stage.failed",
"stage.retrying",
]);
// Single source of truth: every event type the `eventsToActivity` reducer in
// `routes/run-stages.tsx` consumes. When any of these arrive for a stage we
// currently view, the stage-events SWR key for that stage must be invalidated
@ -177,6 +183,7 @@ function resyncKeysForRun(runId: string) {
}
function stageIdFromPayload(payload: RunEventPayload): string | undefined {
if (typeof payload.stage_id === "string") return payload.stage_id;
if (typeof payload.node_id === "string") return payload.node_id;
const nodeId = payload.properties?.node_id;
return typeof nodeId === "string" ? nodeId : undefined;
@ -189,4 +196,4 @@ export function useRunEvents(runId: string | undefined) {
if (!runId) return;
return subscribeToRunEvents(runId, mutate as MutateFn);
}, [mutate, runId]);
}
}

172
apps/fabro-web/app/lib/stage-sidebar.test.ts Normal file
View file

@ -0,0 +1,172 @@
import { describe, expect, test } from "bun:test";
import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client";
import type { Stage } from "../components/stage-sidebar";
import { aggregateGraphNodeStatus, formatStageLabel, mapRunStagesToSidebarStages } from "./stage-sidebar";
function makeStage(nodeId: string, visit: number, status: StageState): Stage {
return {
id: `${nodeId}@${visit}`,
name: nodeId,
nodeId,
visit,
status,
duration: "--",
};
}
describe("mapRunStagesToSidebarStages", () => {
test("maps two visits of the same node to distinct sidebar entries", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "apply-changes@1",
name: "Apply Changes",
status: "succeeded",
duration_secs: 12.5,
node_id: "apply",
visit: 1,
},
{
id: "apply-changes@2",
name: "Apply Changes",
status: "running",
node_id: "apply",
visit: 2,
},
],
meta: { has_more: false },
};
const result = mapRunStagesToSidebarStages(stages);
expect(result).toHaveLength(2);
expect(result[0].id).toBe("apply-changes@1");
expect(result[0].nodeId).toBe("apply");
expect(result[0].visit).toBe(1);
expect(formatStageLabel(result[0])).toBe("Apply Changes");
expect(result[1].id).toBe("apply-changes@2");
expect(result[1].nodeId).toBe("apply");
expect(result[1].visit).toBe(2);
expect(formatStageLabel(result[1])).toBe("Apply Changes (2)");
});
test("filters by node_id (suffixed start@1 / exit@1 are still hidden)", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "start@1",
name: "start",
status: "succeeded",
node_id: "start",
visit: 1,
},
{
id: "verify@1",
name: "verify",
status: "succeeded",
node_id: "verify",
visit: 1,
},
{
id: "exit@1",
name: "exit",
status: "succeeded",
node_id: "exit",
visit: 1,
},
],
meta: { has_more: false },
};
const result = mapRunStagesToSidebarStages(stages);
expect(result.map((s) => s.id)).toEqual(["verify@1"]);
});
test("missing duration renders as '--'", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "verify@1",
name: "verify",
status: "running",
node_id: "verify",
visit: 1,
},
],
meta: { has_more: false },
};
expect(mapRunStagesToSidebarStages(stages)[0].duration).toBe("--");
});
});
describe("aggregateGraphNodeStatus", () => {
test("(failed, running) renders as running and clicks open the latest visit", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "failed"),
makeStage("verify", 2, "running"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "running",
latestStageId: "verify@2",
});
});
test("(failed, succeeded) renders as succeeded — failure-then-fix shows healed", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "failed"),
makeStage("verify", 2, "succeeded"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "succeeded",
latestStageId: "verify@2",
});
});
test("(succeeded, failed) renders as failed and clicks open the latest visit", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "succeeded"),
makeStage("verify", 2, "failed"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "failed",
latestStageId: "verify@2",
});
});
test("(running, retrying) — latest active wins", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "running"),
makeStage("verify", 2, "retrying"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "retrying",
latestStageId: "verify@2",
});
});
test("orders by visit even when input is shuffled", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 2, "running"),
makeStage("verify", 1, "failed"),
]);
expect(result.get("verify")?.latestStageId).toBe("verify@2");
});
test("single visit per node is unaffected", () => {
const result = aggregateGraphNodeStatus([
makeStage("plan", 1, "succeeded"),
makeStage("apply", 1, "running"),
]);
expect(result.get("plan")).toEqual({
displayStatus: "succeeded",
latestStageId: "plan@1",
});
expect(result.get("apply")).toEqual({
displayStatus: "running",
latestStageId: "apply@1",
});
});
});

51
apps/fabro-web/app/lib/stage-sidebar.ts
View file

@ -10,18 +10,65 @@ export const SUCCEEDED_STAGE_STATES: ReadonlySet<StageState> = new Set([
"partially_succeeded",
]);
/**
* Display label for a stage. Suffixes `(N)` for visits > 1 so a looped node
* (e.g. `verify`) renders as `verify`, `verify (2)`, `verify (3)` in the
* sidebar and stage header.
*/
export function formatStageLabel(stage: { name: string; visit: number }): string {
return stage.visit > 1 ? `${stage.name} (${stage.visit})` : stage.name;
}
export function mapRunStagesToSidebarStages(
stagesResult: PaginatedRunStageList | null | undefined,
): Stage[] {
return (stagesResult?.data ?? [])
.filter((stage) => isVisibleStage(stage.id))
.filter((stage) => isVisibleStage(stage.node_id))
.map((stage) => ({
id: stage.id,
name: stage.name,
dotId: stage.dot_id ?? stage.id,
nodeId: stage.node_id,
visit: stage.visit,
status: stage.status,
duration: stage.duration_secs != null
? formatDurationSecs(stage.duration_secs)
: "--",
}));
}
/**
* Aggregate per-node display state for the workflow graph.
*
* Status policy: if any visit is active (running/retrying), the node renders
* that active state (latest active visit wins). Otherwise the node renders
* the latest visit's terminal state. The click target is always the latest
* visit's stageId.
*/
export function aggregateGraphNodeStatus(stages: readonly Stage[]): Map<
string,
{ displayStatus: StageState; latestStageId: string }
> {
// Single pass per nodeId: track the visit with the highest `visit` overall
// (drives click target + terminal status) and the highest-visit *active*
// stage (drives display when any visit is in flight).
const latest = new Map<string, Stage>();
const latestActive = new Map<string, Stage>();
for (const stage of stages) {
const prevLatest = latest.get(stage.nodeId);
if (!prevLatest || stage.visit > prevLatest.visit) {
latest.set(stage.nodeId, stage);
}
if (ACTIVE_STAGE_STATES.has(stage.status)) {
const prevActive = latestActive.get(stage.nodeId);
if (!prevActive || stage.visit > prevActive.visit) {
latestActive.set(stage.nodeId, stage);
}
}
}
const result = new Map<string, { displayStatus: StageState; latestStageId: string }>();
for (const [nodeId, latestStage] of latest) {
const display = latestActive.get(nodeId) ?? latestStage;
result.set(nodeId, { displayStatus: display.status, latestStageId: latestStage.id });
}
return result;
}

34
apps/fabro-web/app/routes/run-overview.tsx
View file

@ -3,7 +3,6 @@ import { useNavigate, useParams } from "react-router";
import { graphTheme } from "../lib/graph-theme";
import { useRun, useRunGraph, useRunStages } from "../lib/queries";
import { StageSidebar } from "../components/stage-sidebar";
import type { Stage } from "../components/stage-sidebar";
import {
GRAPH_DEFAULT_ZOOM_INDEX,
GRAPH_ZOOM_STEPS,
@ -13,6 +12,7 @@ import { EmptyState } from "../components/state";
import {
ACTIVE_STAGE_STATES,
SUCCEEDED_STAGE_STATES,
aggregateGraphNodeStatus,
mapRunStagesToSidebarStages,
} from "../lib/stage-sidebar";
@ -54,27 +54,27 @@ export default function RunOverview() {
const inner = innerRef.current;
if (!inner || !graphSvg) return;
let cancelled = false;
(async () => {
if (cancelled) return;
inner.innerHTML = graphSvg;
const svg = inner.querySelector("svg");
if (!svg) return;
svgRef.current = svg;
const gt = graphTheme;
const runningDotIds = new Set<string>(
stages.filter((s: Stage) => ACTIVE_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id),
);
const failedDotIds = new Set<string>(
stages.filter((s: Stage) => s.status === "failed").map((s: Stage) => s.dotId ?? s.id),
);
const completedDotIds = new Set<string>(
stages.filter((s: Stage) => SUCCEEDED_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id),
);
const dotIdToStageId = new Map<string, string>(
stages.map((s: Stage) => [s.dotId ?? s.id, s.id]),
);
const aggregated = aggregateGraphNodeStatus(stages);
const runningDotIds = new Set<string>();
const failedDotIds = new Set<string>();
const completedDotIds = new Set<string>();
const dotIdToStageId = new Map<string, string>();
for (const [nodeId, { displayStatus, latestStageId }] of aggregated) {
dotIdToStageId.set(nodeId, latestStageId);
if (ACTIVE_STAGE_STATES.has(displayStatus)) {
runningDotIds.add(nodeId);
} else if (displayStatus === "failed") {
failedDotIds.add(nodeId);
} else if (SUCCEEDED_STAGE_STATES.has(displayStatus)) {
completedDotIds.add(nodeId);
}
}
const ns = "http://www.w3.org/2000/svg";
for (const group of svg.querySelectorAll(".node")) {
@ -148,8 +148,6 @@ export default function RunOverview() {
}
}
}
})();
return () => { cancelled = true; };
}, [stages, graphSvg, id, navigate, terminalOutcome]);
const onPointerDown = useCallback((e: React.PointerEvent) => {

82
apps/fabro-web/app/routes/run-stages.test.ts
View file

@ -23,11 +23,53 @@ function envelope(seq: number, partial: Partial<EventEnvelope>): EventEnvelope {
id: `evt-${seq}`,
ts: "2026-04-09T12:00:00Z",
run_id: "run-1",
event: "stage.prompt",
...partial,
} as EventEnvelope;
}
describe("eventsToActivity", () => {
test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "stage.prompt",
stage_id: "verify@1",
node_id: "verify",
properties: { text: "first visit prompt" },
}),
envelope(2, {
event: "stage.prompt",
stage_id: "verify@2",
node_id: "verify",
properties: { text: "second visit prompt" },
}),
envelope(3, {
event: "agent.message",
stage_id: "verify@1",
node_id: "verify",
properties: { text: "first visit reply" },
}),
envelope(4, {
event: "agent.message",
stage_id: "verify@2",
node_id: "verify",
properties: { text: "second visit reply" },
}),
];
const firstVisit = eventsToActivity(events, "verify@1");
expect(firstVisit).toEqual([
{ kind: "system", content: "first visit prompt" },
{ kind: "assistant", content: "first visit reply" },
]);
const secondVisit = eventsToActivity(events, "verify@2");
expect(secondVisit).toEqual([
{ kind: "system", content: "second visit prompt" },
{ kind: "assistant", content: "second visit reply" },
]);
});
test("pairs command.started + command.completed into a single command turn", () => {
const events: EventEnvelope[] = [
envelope(1, {
@ -52,6 +94,7 @@ describe("eventsToActivity", () => {
expect(turns).toHaveLength(1);
expect(turns[0]).toMatchObject({
kind: "command",
stageId: "fmt",
script: "cargo fmt",
language: "shell",
stdout: "ok",
@ -60,6 +103,39 @@ describe("eventsToActivity", () => {
});
});
test("command turn carries the requested stage_id, no @1 fallback", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "command.started",
stage_id: "verify@2",
node_id: "verify",
properties: { script: "echo hi", language: "shell" },
}),
envelope(2, {
event: "command.completed",
stage_id: "verify@2",
node_id: "verify",
properties: {
stdout: "hi",
stderr: "",
exit_code: 0,
duration_ms: 5,
termination: "exited",
},
}),
];
const turns = eventsToActivity(events, "verify@2");
expect(turns).toHaveLength(1);
const turn = turns[0];
expect(turn.kind).toBe("command");
if (turn.kind === "command") {
expect(turn.stageId).toBe("verify@2");
expect(turn.script).toBe("echo hi");
expect(turn.running).toBe(false);
}
});
test("pairs agent.tool.started + agent.tool.completed into a single tool turn", () => {
const events: EventEnvelope[] = [
envelope(1, {
@ -97,9 +173,7 @@ describe("eventsToActivity", () => {
}
});
test("ignores unknown event types and events for other nodes", () => {
// The reducer defensively filters by node_id even though the server scopes
// this endpoint, and only consumes STAGE_ACTIVITY_EVENT_TYPES.
test("ignores unknown event types and events for other stages", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "stage.started",
@ -129,4 +203,4 @@ describe("eventsToActivity", () => {
expect(turns[0].content).toBe("signal");
}
});
});
});

35
apps/fabro-web/app/routes/run-stages.tsx
View file

@ -42,7 +42,7 @@ import { CopyButton } from "../components/ui";
import { formatDurationSecs } from "../lib/format";
import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries";
import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events";
import { mapRunStagesToSidebarStages } from "../lib/stage-sidebar";
import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar";
import { getNumber, getString, type UnknownRecord } from "../lib/unknown";
import {
CommandOutputStream,
@ -72,6 +72,12 @@ function assertNever(value: never): never {
throw new Error(`Unhandled stage activity event type: ${value}`);
}
function activityEventStageId(event: EventEnvelope): string | undefined {
if (typeof event.stage_id === "string") return event.stage_id;
if (typeof event.node_id === "string") return event.node_id;
return getString(event.properties ?? {}, "node_id");
}
export function eventsToActivity(events: EventEnvelope[], stageId: string): TurnType[] {
const turns: TurnType[] = [];
// Collect tool pairs: started → completed
@ -80,11 +86,18 @@ export function eventsToActivity(events: EventEnvelope[], stageId: string): Turn
let pendingCommand: { stageId: string; script: string; language: string } | undefined;
for (const e of events) {
if (e.node_id !== stageId || !STAGE_ACTIVITY_EVENT_SET.has(e.event)) continue;
const eventName = e.event;
if (
activityEventStageId(e) !== stageId ||
!eventName ||
!STAGE_ACTIVITY_EVENT_SET.has(eventName)
) {
continue;
}
// Exhaustive switch over StageActivityEventType: adding a new variant to
// STAGE_ACTIVITY_EVENT_TYPES forces a TS error here until the case is
// handled, keeping the SWR invalidation set and the reducer in sync.
const eventType = e.event as StageActivityEventType;
const eventType = eventName as StageActivityEventType;
const props = e.properties ?? {};
switch (eventType) {
case "stage.prompt":
@ -122,7 +135,7 @@ export function eventsToActivity(events: EventEnvelope[], stageId: string): Turn
}
case "command.started": {
pendingCommand = {
stageId: e.stage_id ?? `${stageId}@1`,
stageId,
script: getString(props, "script") ?? "",
language: getString(props, "language") ?? "shell",
};
@ -131,7 +144,7 @@ export function eventsToActivity(events: EventEnvelope[], stageId: string): Turn
case "command.completed": {
turns.push({
kind: "command",
stageId: pendingCommand?.stageId ?? e.stage_id ?? `${stageId}@1`,
stageId: pendingCommand?.stageId ?? stageId,
script: pendingCommand?.script ?? "",
language: pendingCommand?.language ?? "shell",
stdout: getString(props, "stdout") ?? "",
@ -589,7 +602,7 @@ export default function RunStages() {
: [],
[stageEventsQuery.data, selectedStageId],
);
const isRunning = selectedStage?.status === "running";
const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false;
if (!id || !stages.length) {
return (
@ -611,11 +624,13 @@ export default function RunStages() {
<div className="min-w-0 flex-1 space-y-3">
<div className="sticky top-0 z-10 -mx-2 flex items-center gap-2 bg-page/85 px-2 py-2 backdrop-blur">
<SelectedIcon className={`size-5 ${selectedConfig.color} ${isRunning ? "animate-spin" : ""}`} />
<h3 className="text-base font-semibold text-fg">{selectedStage.name}</h3>
<SelectedIcon className={`size-5 ${selectedConfig.color} ${isActive ? "animate-spin" : ""}`} />
<h3 className="text-base font-semibold text-fg">
{formatStageLabel(selectedStage)}
</h3>
<span className="font-mono text-xs tabular-nums text-fg-muted">
<RunningStageDuration
isRunning={isRunning}
isRunning={isActive}
duration={selectedStage.duration}
/>
</span>
@ -636,4 +651,4 @@ export default function RunStages() {
</div>
</div>
);
}
}

45
docs/public/api-reference/fabro-api.yaml
View file

@ -1911,10 +1911,10 @@ paths:
operationId: listStageEvents
tags: [Run Internals]
summary: List Stage Events
description: Returns a paginated JSON list of stored run events scoped to a single workflow node (stage).
description: Returns a paginated JSON list of stored run events scoped to a single stage visit.
parameters:
- $ref: "#/components/parameters/RunId"
- $ref: "#/components/parameters/StageNodeId"
- $ref: "#/components/parameters/StageId"
- $ref: "#/components/parameters/SinceSeq"
- $ref: "#/components/parameters/EventLimit"
responses:
@ -2931,15 +2931,6 @@ components:
type: string
example: code@2
StageNodeId:
name: stageId
in: path
required: true
description: Workflow node id (matches RunStage.id; not visit-qualified).
schema:
type: string
example: detect-drift
CommandLogStream:
name: stream
in: path
@ -6155,7 +6146,7 @@ components:
example: 3501.0
BillingStageRef:
description: Reference to a billing stage.
description: Reference to a workflow node in a billing stage row.
type: object
required:
- id
@ -6315,11 +6306,13 @@ components:
- id
- name
- status
- node_id
- visit
properties:
id:
type: string
description: Unique stage identifier within the run.
example: propose-changes
description: StageId in "node_id@visit" form, e.g. verify@2.
example: verify@2
name:
type: string
description: Human-readable stage name.
@ -6330,10 +6323,16 @@ components:
type: number
description: Time spent in this stage, in seconds.
example: 154.0
dot_id:
node_id:
type: string
description: Node identifier in the Graphviz graph source.
example: propose
description: Node id in the workflow graph; multiple stages with different visits share the same node_id.
example: verify
visit:
type: integer
format: uint32
minimum: 1
description: 1-based visit count; bumped each time the workflow re-enters this node.
example: 2
# ── File Diff Schemas ──────────────────────────────────────────────
@ -6506,7 +6505,7 @@ components:
# ── Billing Schemas ──────────────────────────────────────────────────
RunBillingStage:
description: Token counts and billed totals for a single stage within a run.
description: Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node.
type: object
required:
- stage
@ -6517,7 +6516,7 @@ components:
stage:
$ref: "#/components/schemas/BillingStageRef"
model:
description: Model used for this stage; null for non-LLM stages.
description: Latest usage-bearing visit model for this node; null when no visit used an LLM model.
oneOf:
- $ref: "#/components/schemas/ModelReference"
- type: "null"
@ -6525,7 +6524,7 @@ components:
$ref: "#/components/schemas/BilledTokenCounts"
runtime_secs:
type: number
description: Wall-clock runtime in seconds.
description: Wall-clock runtime in seconds, summed across every visit of this node.
example: 154.0
RunBillingTotals:
@ -6586,7 +6585,7 @@ components:
$ref: "#/components/schemas/ModelReference"
stages:
type: integer
description: Number of stages that used this model.
description: Number of usage-bearing stage visits that used this model.
example: 2
billing:
$ref: "#/components/schemas/BilledTokenCounts"
@ -6601,7 +6600,7 @@ components:
properties:
stages:
type: array
description: Per-stage billing breakdown.
description: Per-node billing breakdown. Each row sums billing and runtime across all visits of that node.
items:
$ref: "#/components/schemas/RunBillingStage"
totals:
@ -8221,4 +8220,4 @@ components:
login:
type: string
description: User's login identifier (e.g. GitHub username).
example: octocat
example: octocat

106
docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md Normal file
View file

@ -0,0 +1,106 @@
# Cross-Tab SSE Coordination Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Coordinate Fabro web SSE subscriptions across tabs so one browser profile/origin opens at most one UI-owned `/api/v1/attach` EventSource in steady state when `BroadcastChannel` is available; brief overlap during election/takeover is tolerated and deduped.
**Architecture:** Add a browser-side SSE coordinator that elects one tab as leader, has that leader own the global EventSource, and broadcasts parsed run events to follower tabs over `BroadcastChannel`. Existing board and run-detail invalidation logic becomes a consumer of that global event feed, with the current per-tab SSE behavior preserved as a compatibility fallback.
**Tech Stack:** React, SWR, browser `BroadcastChannel`, browser `EventSource`, Bun tests, existing Fabro web API query keys.
---
## Summary
Build a browser-side SSE coordinator so Fabro web opens at most one `/api/v1/attach` EventSource per origin/browser profile in steady state when `BroadcastChannel` is available. Temporary duplicate leaders may exist during election/takeover, but event dedupe and generation checks make the overlap harmless and short-lived. The global stream becomes a shared cache-invalidation feed for both the runs board and run detail pages. No server API, OpenAPI, or Rust streaming contract changes are part of v1.
This supersedes the earlier web SSE limitation documented in `docs/plans/2026-04-19-002-feat-web-ui-lifecycle-actions-plan.md`: the old shared hook was code reuse only; this plan adds actual socket deduplication.
## Implementation Changes
- Add `apps/fabro-web/app/lib/cross-tab-sse.ts`.
- Export `subscribeToCrossTabSse(...)` with the same invalidation style as `subscribeToSharedEventSource`, a `resyncKeys` callback for gap recovery, and a `fallbackSubscribe` callback used when cross-tab coordination is unavailable.
- Use `BroadcastChannel` name `fabro:sse:v1`.
- Generate `tabId` with `crypto.randomUUID()` and a safe random fallback.
- Open one leader-owned `EventSource` to `queryKeys.system.attach()` (`/api/v1/attach`) in steady state.
- Leader dispatches each parsed `EventEnvelope` locally and broadcasts it to follower tabs.
- Followers do not open EventSource while a valid visible leader heartbeat exists; lower lexical `tabId` does not preempt a healthy visible leader.
- Implement leader election in the cross-tab module.
- Constants: `HEARTBEAT_MS = 1000`, `LEADER_STALE_MS = 4000`, `ELECTION_JITTER_MS = 150`.
- Messages: `hello`, `heartbeat`, `candidate`, `leader-changed`, `release`, `resync`, `event`.
- Define a typed message union. Every message includes `type`, `version: 1`, `tabId`, and `sentAt`.
- `heartbeat`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`.
- `candidate`: `{ type, version, tabId, sentAt, candidateId: tabId, candidateGeneration, visibility, observedLeaderId, observedGeneration, reason }`, where `reason` is `"hidden-leader" | "stale-leader" | "release" | "no-leader"`.
- `leader-changed`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`.
- `release`: `{ type, version, tabId, sentAt, leaderId, generation }`.
- `resync`: `{ type, version, tabId, sentAt, leaderId, generation, reason }`.
- `event`: `{ type, version, tabId, sentAt, leaderId, generation, payload }`.
- Use the candidacy phase for all leadership changes: hidden-leader takeover, stale-leader recovery, leader release, and no-leader startup.
- A candidate sets `candidateGeneration = observedGeneration + 1`, broadcasts `candidate`, waits jitter, and opens EventSource only if no higher-priority candidate for the same `candidateGeneration` appears.
- Candidate priority is election-scoped: visible candidates outrank hidden candidates; for equal visibility, lower lexical `candidateId` wins. This priority resolves elections and same-generation split brain only; it is not a reason to preempt a fresh visible leader.
- When a visible follower observes a fresh hidden leader heartbeat, it enters candidacy with `reason: "hidden-leader"`.
- When tabs detect a stale leader, leader release, or no known leader, they enter the same candidacy flow with the matching `reason`.
- If two visible candidates race for the same observed leader/generation, the lower lexical `candidateId` wins.
- Current leaders release when they observe a candidate whose `observedLeaderId` matches their `leaderId` and whose `observedGeneration` is current or newer.
- If same-generation split brain still occurs, lower-priority leaders release when they observe a same-generation higher-priority leader heartbeat or `leader-changed`.
- Each new leader uses `candidateGeneration`, broadcasts `leader-changed`, and followers ignore heartbeats/events from non-current leaders or stale generations.
- Hidden leader keeps the stream only when no visible candidate takes over.
- On `pagehide`/last local unsubscribe, a leader closes EventSource and broadcasts `release`.
- Brief split brain is tolerated; dedupe events by `payload.id`, falling back to `${run_id}:${seq}:${event}`.
- Keep dedupe bounded with a recent-event cache: max 1000 IDs and 5 minute TTL. Evict oldest entries when the max is exceeded and prune expired entries during event handling. Duplicate invalidations after eviction are acceptable; unbounded growth is not.
- Migrate consumers.
- `apps/fabro-web/app/lib/board-events.ts`: subscribe through the cross-tab global stream; keep existing board event allowlist.
- `apps/fabro-web/app/lib/run-events.ts`: subscribe through the same global stream, filter by `payload.run_id === runId`, and reuse `queryKeysForRunEvent`.
- In coordinated mode, run detail pages stay subscribed while mounted, including terminal runs, so post-terminal archive/unarchive changes can reconcile live.
- Do not close the global stream on `run.completed` / `run.failed`; terminal events only invalidate run-scoped keys.
- Keep `subscribeToSharedEventSource` in `apps/fabro-web/app/lib/sse.ts` for fallback and existing local sharing behavior.
- Gap and fallback behavior.
- If `BroadcastChannel` is unavailable or throws, call each subscriber's `fallbackSubscribe`.
- Board fallback uses the existing global `/api/v1/attach` path.
- Run detail fallback preserves the existing run-scoped `/api/v1/runs/:id/attach` path, so the old terminal-tab stale limitation remains only in fallback mode.
- Do not add replay to `/api/v1/attach`.
- On leader takeover, stale leader timeout, leader release, and new leader generation, broadcast `resync` or `leader-changed` so every tab with active local subscriptions runs its own `resyncKeys`.
- On `visibilitychange` back to visible without leadership change, run only that tab's local `resyncKeys`; do not broadcast cross-tab resync.
- Board `resyncKeys`: `queryKeys.boards.runs()`.
- Run `resyncKeys`: detail, files, billing, stages, events, LR graph, TB graph, and questions for that run.
## Tests
- Add `apps/fabro-web/app/lib/cross-tab-sse.test.ts` with fake `BroadcastChannel`, fake `EventSource`, and fake timers.
- One leader opens `/api/v1/attach`; followers open no EventSource.
- Leader broadcasts an event and all local subscribers receive invalidations.
- Run subscribers ignore events for other `run_id` values.
- Board and run subscriptions can coexist on the same global stream.
- Temporary duplicate leaders are allowed only during election/takeover and converge back to one leader.
- With fresh hidden-leader heartbeats, a visible tab broadcasts candidacy, hidden leader closes, visible tab opens `/api/v1/attach`, and followers resync.
- Two visible candidates racing for the same hidden leader resolve to the lower lexical `candidateId`.
- Two tabs detect the same stale leader simultaneously; only the winning candidate opens `/api/v1/attach` after jitter.
- Same-generation split brain converges to one leader by visibility, then lexical `tabId`.
- A fresh visible leader exists; a new visible follower with a lower lexical `tabId` joins and does not take leadership.
- A hidden leader does not release for a candidate that references an old `observedLeaderId` or stale `observedGeneration`.
- Stale heartbeat triggers takeover and every active tab calls its own `resyncKeys`.
- A follower tab calls its own board/run `resyncKeys` after another tab becomes leader.
- Duplicate event IDs are ignored.
- The recent-event dedupe cache evicts by TTL and max-size bound; duplicate invalidation may recur only after eviction.
- Stale heartbeat/event messages from an old leader/generation are ignored after takeover.
- Last unsubscribe closes leader EventSource and sends `release`.
- Missing/broken BroadcastChannel uses the per-tab fallback.
- Update existing tests.
- `apps/fabro-web/app/lib/board-events.test.tsx`: assert coordinated mode uses `/api/v1/attach` once and fallback preserves current behavior.
- `apps/fabro-web/app/lib/run-events.test.tsx`: assert coordinated mode filters by `run_id`; fallback preserves current `/runs/:id/attach` behavior; terminal run events do not close the global coordinator; terminal detail tabs still receive archive/unarchive invalidations while mounted in coordinated mode.
- Verification commands.
- `cd apps/fabro-web && bun test app/lib/cross-tab-sse.test.ts app/lib/board-events.test.tsx app/lib/run-events.test.tsx`
- `cd apps/fabro-web && bun run typecheck`
- Optional manual check: open 8 run-detail tabs for active runs; after election settles and with BroadcastChannel available, Chrome should show one active `/api/v1/attach` EventStream across the participating tabs and no UI-created `/api/v1/runs/:id/attach` streams.
## Assumptions
- Treat SSE as live cache invalidation, not an exact gapless event log.
- Preserve current run-specific attach endpoint for CLI, API clients, and fallback only.
- Do not touch unrelated dirty worktree files; intentionally replace or adapt the earlier interrupted SSE test edits as part of the new test suite.
- No docs or public API updates are required for v1 because this is an internal web transport change.

7
lib/crates/fabro-dump/src/lib.rs
View file

@ -66,18 +66,13 @@ impl RunDump {
entries.push(RunDumpEntry::text("graph.fabro", graph_source.clone()));
}
let mut stages: Vec<_> = state.iter_stages().collect();
let stages: Vec<_> = state.iter_stages().collect();
if stages.len() > MAX_STAGES_IN_DUMP {
bail!(
"run dump supports at most {MAX_STAGES_IN_DUMP} stages with the current path prefix width (got {})",
stages.len()
);
}
stages.sort_by(|(left_id, left), (right_id, right)| {
left.first_event_seq
.cmp(&right.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
let mut stage_ranks = HashMap::new();
for (index, (stage_id, _)) in stages.iter().enumerate() {

75
lib/crates/fabro-server/src/demo/mod.rs
View file

@ -24,7 +24,7 @@ use serde_json::json;
use crate::error::ApiError;
use crate::principal_middleware::RequiredUser;
use crate::run_selector::{ResolveRunError, resolve_run_by_selector};
use crate::server::{AppState, EventListParams, PaginationParams};
use crate::server::{AppState, EventListParams, PaginationParams, parse_stage_id_path};
fn paginated_response<T: serde::Serialize>(
items: Vec<T>,
@ -140,13 +140,20 @@ pub(crate) async fn get_stage_events(
Path((_id, stage_id)): Path<(String, String)>,
Query(params): Query<EventListParams>,
) -> Response {
let stage_id = match parse_stage_id_path(&stage_id) {
Ok(stage_id) => stage_id,
Err(response) => return response,
};
let since_seq = params.since_seq();
let limit = params.limit();
let mut matches: Vec<EventEnvelope> = runs::stage_events()
.into_iter()
.filter(|envelope| {
envelope.seq >= since_seq
&& envelope.event.node_id.as_deref() == Some(stage_id.as_str())
&& (envelope.event.stage_id.as_ref() == Some(&stage_id)
|| (envelope.event.stage_id.is_none()
&& stage_id.visit() == 1
&& envelope.event.node_id.as_deref() == Some(stage_id.node_id())))
})
.take(limit + 1)
.collect();
@ -804,9 +811,10 @@ mod runs {
RunNamespace, RunPrepareSettings, RunSandboxSettings,
};
use fabro_types::settings::{InterpString, ProjectNamespace, WorkflowNamespace};
use fabro_types::{RunId, WorkflowSettings};
use fabro_types::{RunId, StageId, WorkflowSettings};
use super::ts;
use crate::server::run_stage_from_stage_id;
fn labels(entries: &[(&str, &str)]) -> HashMap<String, String> {
entries
@ -1201,34 +1209,36 @@ mod runs {
pub(super) fn stages() -> Vec<RunStage> {
vec![
RunStage {
id: "detect-drift".into(),
name: "Detect Drift".into(),
status: StageState::Succeeded,
duration_secs: Some(72.0),
dot_id: Some("detect".into()),
},
RunStage {
id: "propose-changes".into(),
name: "Propose Changes".into(),
status: StageState::Succeeded,
duration_secs: Some(154.0),
dot_id: Some("propose".into()),
},
RunStage {
id: "review-changes".into(),
name: "Review Changes".into(),
status: StageState::Succeeded,
duration_secs: Some(45.0),
dot_id: Some("review".into()),
},
RunStage {
id: "apply-changes".into(),
name: "Apply Changes".into(),
status: StageState::Running,
duration_secs: Some(118.0),
dot_id: Some("apply".into()),
},
run_stage_from_stage_id(
&StageId::new("detect-drift", 1),
"Detect Drift",
StageState::Succeeded,
Some(72.0),
),
run_stage_from_stage_id(
&StageId::new("propose-changes", 1),
"Propose Changes",
StageState::Succeeded,
Some(154.0),
),
run_stage_from_stage_id(
&StageId::new("review-changes", 1),
"Review Changes",
StageState::Succeeded,
Some(45.0),
),
run_stage_from_stage_id(
&StageId::new("apply-changes", 1),
"Apply Changes",
StageState::Succeeded,
Some(118.0),
),
run_stage_from_stage_id(
&StageId::new("apply-changes", 2),
"Apply Changes",
StageState::Running,
None,
),
]
}
@ -1242,6 +1252,7 @@ mod runs {
let run_id = demo_run_id(1);
let node_id = "detect-drift";
let stage_id = fabro_types::StageId::new(node_id, 1);
let ts = ts("2026-03-06T14:30:00Z");
let make_envelope = |seq: u32, id: &str, body: EventBody| EventEnvelope {
@ -1252,7 +1263,7 @@ mod runs {
run_id,
node_id: Some(node_id.into()),
node_label: Some("Detect Drift".into()),
stage_id: None,
stage_id: Some(stage_id.clone()),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,

127
lib/crates/fabro-server/src/server.rs
View file

@ -54,7 +54,7 @@ use fabro_llm::types::{
ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Role, ToolChoice,
ToolDefinition,
};
use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog, ModelTestMode, Provider};
use fabro_model::{BilledTokenCounts, Catalog, ModelTestMode, Provider};
use fabro_redact::redact_jsonl_line;
use fabro_sandbox::daytona::{self, DaytonaSandbox};
use fabro_sandbox::reconnect::reconnect;
@ -537,17 +537,48 @@ pub(crate) struct ResolvedAppStateSettings {
pub(crate) manifest_run_settings: std::result::Result<RunNamespace, SharedError>,
}
fn accumulate_model_billing(entry: &mut ModelBillingTotals, usage: &BilledModelUsage) {
let tokens = usage.tokens();
entry.stages += 1;
entry.billing.input_tokens += tokens.input_tokens;
entry.billing.output_tokens += tokens.output_tokens;
entry.billing.reasoning_tokens += tokens.reasoning_tokens;
entry.billing.cache_read_tokens += tokens.cache_read_tokens;
entry.billing.cache_write_tokens += tokens.cache_write_tokens;
entry.billing.total_tokens += tokens.total_tokens();
if let Some(value) = usage.total_usd_micros {
*entry.billing.total_usd_micros.get_or_insert(0) += value;
fn accumulate_billed_token_counts(target: &mut BilledTokenCounts, source: &BilledTokenCounts) {
target.input_tokens += source.input_tokens;
target.output_tokens += source.output_tokens;
target.reasoning_tokens += source.reasoning_tokens;
target.cache_read_tokens += source.cache_read_tokens;
target.cache_write_tokens += source.cache_write_tokens;
target.total_tokens += source.total_tokens;
if let Some(value) = source.total_usd_micros {
*target.total_usd_micros.get_or_insert(0) += value;
}
}
fn accumulate_billing_rollup(
accumulator: &mut BillingAccumulator,
rollup: &fabro_workflow::ProjectionBillingRollup,
) {
accumulator.total_runs += 1;
accumulator.total_runtime_secs += rollup.runtime_ms as f64 / 1000.0;
for model in &rollup.by_model {
let entry = accumulator
.by_model
.entry(model.model_id.clone())
.or_default();
entry.stages += model.stages;
accumulate_billed_token_counts(&mut entry.billing, &model.billing);
}
}
pub(crate) fn run_stage_from_stage_id(
stage_id: &StageId,
name: impl Into<String>,
status: StageState,
duration_secs: Option<f64>,
) -> RunStage {
RunStage {
id: stage_id.to_string(),
name: name.into(),
status,
duration_secs,
node_id: stage_id.node_id().to_string(),
visit: std::num::NonZeroU32::new(stage_id.visit())
.expect("StageId stores a non-zero visit"),
}
}
@ -2777,9 +2808,9 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
}
}
// Save final checkpoint
let checkpoint = match run_store.state().await {
Ok(state) => state.checkpoint,
// Save final projection
let final_projection = match run_store.state().await {
Ok(state) => Some(state),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state from store");
None
@ -2787,32 +2818,17 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
// Accumulate aggregate usage after execution completes.
if let Some(ref cp) = checkpoint {
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store");
HashMap::default()
}
};
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
agg.total_runs += 1;
let mut run_runtime: f64 = 0.0;
for (node_id, outcome) in &cp.node_outcomes {
if let Some(usage) = &outcome.usage {
let entry = agg
.by_model
.entry(usage.model_id().to_string())
.or_default();
accumulate_model_billing(entry, usage);
}
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
run_runtime += duration_ms as f64 / 1000.0;
if let Some(ref projection) = final_projection {
if projection.checkpoint.is_some() {
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
accumulate_billing_rollup(
&mut agg,
&fabro_workflow::billing_rollup_from_projection(projection),
);
}
agg.total_runtime_secs += run_runtime;
}
let mut runs = state.runs.lock().expect("runs lock poisoned");
@ -2861,7 +2877,9 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
}
}
managed_run.checkpoint = checkpoint;
managed_run.checkpoint = final_projection
.as_ref()
.and_then(|projection| projection.checkpoint.clone());
managed_run.run_dir = Some(run_dir);
clear_live_run_state(managed_run);
}
@ -3104,32 +3122,15 @@ async fn execute_run_subprocess(state: Arc<AppState>, run_id: RunId) {
}
};
if let Some(ref checkpoint) = final_state.checkpoint {
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store");
HashMap::default()
}
};
if final_state.checkpoint.is_some() {
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
agg.total_runs += 1;
let mut run_runtime: f64 = 0.0;
for (node_id, outcome) in &checkpoint.node_outcomes {
if let Some(usage) = &outcome.usage {
let entry = agg
.by_model
.entry(usage.model_id().to_string())
.or_default();
accumulate_model_billing(entry, usage);
}
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
run_runtime += duration_ms as f64 / 1000.0;
}
agg.total_runtime_secs += run_runtime;
accumulate_billing_rollup(
&mut agg,
&fabro_workflow::billing_rollup_from_projection(&final_state),
);
}
let mut runs = state.runs.lock().expect("runs lock poisoned");

277
lib/crates/fabro-server/src/server/handler/billing.rs
View file

@ -1,13 +1,13 @@
use std::sync::Arc;
use fabro_types::EventBody;
use fabro_store::RunProjectionReducer;
use fabro_types::{EventBody, RunProjection, StageId};
use super::super::{
ApiError, AppState, BilledTokenCounts, BillingByModel, BillingStageRef, EventEnvelope, HashMap,
IntoResponse, Json, ListResponse, ModelBillingTotals, ModelReference, PaginationParams, Path,
Query, RequiredUser, Response, Router, RunBilling, RunBillingStage, RunBillingTotals, RunId,
RunStage, RunStatus, StageState, State, StatusCode, accumulate_model_billing, get,
parse_run_id_path,
ApiError, AppState, BillingByModel, BillingStageRef, EventEnvelope, HashMap, IntoResponse,
Json, ListResponse, ModelReference, PaginationParams, Path, Query, RequiredUser, Response,
Router, RunBilling, RunBillingStage, RunBillingTotals, RunId, StageState, State, StatusCode,
get, parse_run_id_path, run_stage_from_stage_id,
};
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -16,25 +16,41 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
.route("/runs/{id}/billing", get(get_run_billing))
}
fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState {
let latest = events.iter().rev().find(|envelope| {
envelope.event.node_id.as_deref() == Some(node_id)
&& matches!(
&envelope.event.body,
EventBody::StageRetrying(_)
| EventBody::StageStarted(_)
| EventBody::StageCompleted(_)
| EventBody::StageFailed(_)
)
});
if latest.is_some_and(|e| matches!(&e.event.body, EventBody::StageRetrying(_))) {
StageState::Retrying
} else {
StageState::Running
/// Map a `stage.*` lifecycle event body to the [`StageState`] it implies.
/// Returns `None` for any other variant.
fn stage_state_from_lifecycle(body: &EventBody) -> Option<StageState> {
match body {
EventBody::StageStarted(_) => Some(StageState::Running),
EventBody::StageRetrying(_) => Some(StageState::Retrying),
EventBody::StageFailed(props) => Some(if props.will_retry {
StageState::Retrying
} else {
StageState::Failed
}),
EventBody::StageCompleted(props) => Some(StageState::from(props.status)),
_ => None,
}
}
/// Single-pass scan over `events` building the latest [`StageState`] for each
/// [`StageId`] from lifecycle events (started/retrying/completed/failed). Each
/// later lifecycle event overwrites earlier ones, leaving the latest as the
/// stored value — equivalent to "scan in reverse, take first match" but in O(E)
/// for the whole list rather than O(stages × events).
fn latest_stage_states(events: &[EventEnvelope]) -> HashMap<StageId, StageState> {
let mut states = HashMap::new();
for envelope in events {
let Some(stage_id) = envelope.event.stage_id.as_ref() else {
continue;
};
let Some(state) = stage_state_from_lifecycle(&envelope.event.body) else {
continue;
};
states.insert(stage_id.clone(), state);
}
states
}
async fn list_run_stages(
_auth: RequiredUser,
State(state): State<Arc<AppState>>,
@ -46,80 +62,41 @@ async fn list_run_stages(
Err(response) => return response,
};
// Try live run first.
let (checkpoint, run_is_active) = {
let runs = state.runs.lock().expect("runs lock poisoned");
match runs.get(&id) {
Some(managed_run) => {
let active = !matches!(
managed_run.status,
RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead
);
(managed_run.checkpoint.clone(), active)
}
None => (None, false),
}
};
// Fall back to stored run.
let (checkpoint, run_is_active) = if checkpoint.is_some() {
(checkpoint, run_is_active)
} else {
match state.store.open_run_reader(&id).await {
Ok(run_store) => match run_store.state().await {
Ok(run_state) => {
let active = run_state.status.is_some_and(|status| !status.is_terminal());
(run_state.checkpoint, active)
}
Err(_) => (None, false),
},
Err(_) => return ApiError::not_found("Run not found.").into_response(),
}
};
let Some(checkpoint) = checkpoint else {
return (
StatusCode::OK,
Json(ListResponse::new(Vec::<RunStage>::new())),
)
.into_response();
};
let events = match state.store.open_run_reader(&id).await {
Ok(run_store) => run_store.list_events().await.unwrap_or_default(),
Err(_) => Vec::new(),
Err(_) => return ApiError::not_found("Run not found.").into_response(),
};
let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events);
let projection = match RunProjection::apply_events(&events) {
Ok(projection) => projection,
Err(err) => {
tracing::warn!(
run_id = %id,
error = %err,
"Failed to build run projection; returning empty stages list",
);
RunProjection::default()
}
};
let stage_durations = fabro_workflow::extract_stage_durations_by_stage_id(&events);
let lifecycle_states = latest_stage_states(&events);
let mut stages = Vec::new();
for node_id in &checkpoint.completed_nodes {
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
let status = match checkpoint.node_outcomes.get(node_id) {
Some(outcome) => StageState::from(outcome.status),
None => StageState::Succeeded,
};
stages.push(RunStage {
id: node_id.clone(),
name: node_id.clone(),
status,
duration_secs: Some(duration_ms as f64 / 1000.0),
dot_id: Some(node_id.clone()),
for (stage_id, stage_projection) in projection.iter_stages() {
// Prefer the latest lifecycle event; fall back to the projection's
// stored completion (e.g. for runs recovered from snapshot only).
let status = lifecycle_states.get(stage_id).copied().unwrap_or_else(|| {
stage_projection
.completion
.as_ref()
.map_or(StageState::Pending, |c| StageState::from(c.outcome))
});
}
// Add next node as running if the run is still active.
// The checkpoint's current_node is the last *completed* stage; next_node_id
// is the stage that is currently executing.
if let Some(next_id) = &checkpoint.next_node_id {
if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) {
stages.push(RunStage {
id: next_id.clone(),
name: next_id.clone(),
status: active_stage_state_from_events(&events, next_id),
duration_secs: None,
dot_id: Some(next_id.clone()),
});
}
stages.push(run_stage_from_stage_id(
stage_id,
stage_id.node_id().to_string(),
status,
stage_durations.get(stage_id).map(|ms| *ms as f64 / 1000.0),
));
}
(StatusCode::OK, Json(ListResponse::new(stages))).into_response()
@ -137,91 +114,39 @@ async fn get_run_billing(
}
};
let checkpoint = match run_store.state().await {
Ok(state) => state.checkpoint,
let projection = match run_store.state().await {
Ok(state) => state,
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
let Some(checkpoint) = checkpoint else {
let empty = RunBilling {
by_model: Vec::new(),
stages: Vec::new(),
totals: RunBillingTotals {
cache_read_tokens: 0,
cache_write_tokens: 0,
input_tokens: 0,
output_tokens: 0,
reasoning_tokens: 0,
runtime_secs: 0.0,
total_tokens: 0,
total_usd_micros: None,
let rollup = fabro_workflow::billing_rollup_from_projection(&projection);
let by_model = rollup
.by_model
.iter()
.map(|model| BillingByModel {
billing: model.billing.clone(),
model: ModelReference {
id: model.model_id.clone(),
},
};
return (StatusCode::OK, Json(empty)).into_response();
};
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
let mut by_model_totals = HashMap::<String, ModelBillingTotals>::new();
let mut billed_usages = Vec::new();
let mut runtime_secs = 0.0_f64;
let mut stages = Vec::new();
for node_id in &checkpoint.completed_nodes {
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
runtime_secs += duration_ms as f64 / 1000.0;
let usage = checkpoint
.node_outcomes
.get(node_id)
.and_then(|outcome| outcome.usage.as_ref());
let (billing, model) = if let Some(usage) = usage {
billed_usages.push(usage.clone());
let tokens = usage.tokens();
let billing = BilledTokenCounts {
cache_read_tokens: tokens.cache_read_tokens,
cache_write_tokens: tokens.cache_write_tokens,
input_tokens: tokens.input_tokens,
output_tokens: tokens.output_tokens,
reasoning_tokens: tokens.reasoning_tokens,
total_tokens: tokens.total_tokens(),
total_usd_micros: usage.total_usd_micros,
};
let model_id = usage.model_id().to_string();
accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage);
(billing, Some(ModelReference { id: model_id }))
} else {
(BilledTokenCounts::default(), None)
};
stages.push(RunBillingStage {
billing,
model,
runtime_secs: duration_ms as f64 / 1000.0,
stage: BillingStageRef {
id: node_id.clone(),
name: node_id.clone(),
stages: model.stages,
})
.collect::<Vec<_>>();
let stages = rollup
.stages
.iter()
.map(|stage| RunBillingStage {
billing: stage.billing.clone(),
model: stage
.model_id
.as_ref()
.map(|id| ModelReference { id: id.clone() }),
runtime_secs: stage.duration_ms as f64 / 1000.0,
stage: BillingStageRef {
id: stage.node_id.clone(),
name: stage.node_id.clone(),
},
});
}
let totals = BilledTokenCounts::from_billed_usage(&billed_usages);
let by_model = by_model_totals
.into_iter()
.map(|(model, totals)| BillingByModel {
billing: totals.billing,
model: ModelReference { id: model },
stages: totals.stages,
})
.collect::<Vec<_>>();
@ -229,14 +154,14 @@ async fn get_run_billing(
by_model,
stages,
totals: RunBillingTotals {
cache_read_tokens: totals.cache_read_tokens,
cache_write_tokens: totals.cache_write_tokens,
input_tokens: totals.input_tokens,
output_tokens: totals.output_tokens,
reasoning_tokens: totals.reasoning_tokens,
runtime_secs,
total_tokens: totals.total_tokens,
total_usd_micros: totals.total_usd_micros,
cache_read_tokens: rollup.totals.cache_read_tokens,
cache_write_tokens: rollup.totals.cache_write_tokens,
input_tokens: rollup.totals.input_tokens,
output_tokens: rollup.totals.output_tokens,
reasoning_tokens: rollup.totals.reasoning_tokens,
runtime_secs: rollup.runtime_ms as f64 / 1000.0,
total_tokens: rollup.totals.total_tokens,
total_usd_micros: rollup.totals.total_usd_micros,
},
};

83
lib/crates/fabro-server/src/server/handler/events.rs
View file

@ -5,7 +5,8 @@ use super::super::{
EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path,
Query, RequireRunScoped, RequireRunStageScoped, RequiredUser, Response, Router, RunEvent,
RunId, RunStatus, Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get,
mpsc, parse_run_id_path, redact_jsonl_line, reject_if_archived, update_live_run_from_event,
mpsc, parse_run_id_path, parse_stage_id_path, redact_jsonl_line, reject_if_archived,
update_live_run_from_event,
};
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -209,11 +210,15 @@ async fn list_run_stage_events(
State(state): State<Arc<AppState>>,
Query(params): Query<EventListParams>,
) -> Response {
let stage_id = match parse_stage_id_path(&stage_id) {
Ok(stage_id) => stage_id,
Err(response) => return response,
};
let since_seq = params.since_seq();
let limit = params.limit();
match state.store.open_run_reader(&id).await {
Ok(run_store) => match run_store
.list_events_for_node_from_with_limit(&stage_id, since_seq, limit)
.list_events_for_stage_from_with_limit(&stage_id, since_seq, limit)
.await
{
Ok(mut events) => {
@ -396,6 +401,15 @@ mod stage_events_tests {
}
fn make_event(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload {
make_event_with_stage_id(run_id, idx, node_id, None)
}
fn make_event_with_stage_id(
run_id: &RunId,
idx: u32,
node_id: Option<&str>,
stage_id: Option<&str>,
) -> EventPayload {
let mut value = json!({
"id": format!("evt-{idx}"),
"ts": "2026-04-09T12:00:00Z",
@ -412,6 +426,12 @@ mod stage_events_tests {
.unwrap()
.insert("node_id".into(), json!(node));
}
if let Some(stage_id) = stage_id {
value
.as_object_mut()
.unwrap()
.insert("stage_id".into(), json!(stage_id));
}
EventPayload::new(value, run_id).expect("event payload should validate")
}
@ -460,7 +480,7 @@ mod stage_events_tests {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha/events"
"/api/v1/runs/{run_id}/stages/alpha@1/events"
)))
.await
.unwrap();
@ -478,7 +498,7 @@ mod stage_events_tests {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha/events?since_seq=203"
"/api/v1/runs/{run_id}/stages/alpha@1/events?since_seq=203"
)))
.await
.unwrap();
@ -499,7 +519,7 @@ mod stage_events_tests {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha/events?limit=1"
"/api/v1/runs/{run_id}/stages/alpha@1/events?limit=1"
)))
.await
.unwrap();
@ -517,7 +537,7 @@ mod stage_events_tests {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/unknown-stage/events"
"/api/v1/runs/{run_id}/stages/unknown-stage@1/events"
)))
.await
.unwrap();
@ -538,7 +558,7 @@ mod stage_events_tests {
let absent = RunId::new();
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{absent}/stages/alpha/events"
"/api/v1/runs/{absent}/stages/alpha@1/events"
)))
.await
.unwrap();
@ -565,7 +585,7 @@ mod stage_events_tests {
let request = Request::builder()
.method("GET")
.uri(format!("/api/v1/runs/{run_id}/stages/alpha/events"))
.uri(format!("/api/v1/runs/{run_id}/stages/alpha@1/events"))
.header(header::ACCEPT, "application/json")
.body(Body::empty())
.unwrap();
@ -573,4 +593,51 @@ mod stage_events_tests {
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn returns_only_requested_visit_when_stage_id_is_present() {
let state = test_app_state();
let app = build_test_router(state.clone());
let run_id = RunId::new();
let run_store = state
.store_ref()
.create_run(&run_id)
.await
.expect("test run should be creatable");
run_store
.append_event(&make_event_with_stage_id(
&run_id,
1,
Some("verify"),
Some("verify@1"),
))
.await
.expect("append should succeed");
run_store
.append_event(&make_event_with_stage_id(
&run_id,
2,
Some("verify"),
Some("verify@2"),
))
.await
.expect("append should succeed");
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/verify@2/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let seqs: Vec<u64> = body["data"]
.as_array()
.unwrap()
.iter()
.map(|e| e["seq"].as_u64().unwrap())
.collect();
assert_eq!(seqs, vec![2]);
}
}

799
lib/crates/fabro-server/src/server/tests.rs
View file

@ -18,8 +18,8 @@ use fabro_model::Provider;
use fabro_types::settings::ServerAuthMethod;
use fabro_types::{
AttrValue, AuthMethod, CommandTermination, FailureCategory, FailureDetail, Graph,
InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, StageOutcome,
SystemActorKind, fixtures,
InterviewQuestionRecord, Outcome, QuestionType, RunBlobId, RunId, RunSpec, SystemActorKind,
fixtures,
};
use fabro_util::check_report::CheckStatus;
use httpmock::Method::{GET, POST};
@ -2112,6 +2112,30 @@ async fn create_durable_run_with_events(
}
}
/// Append a stage lifecycle event with an explicit `StageScope`, so the
/// stored envelope carries the full `stage_id` (`node_id@visit`). The bare
/// [`workflow_event::append_event`] helper only writes `node_id` because
/// stage lifecycle variants don't carry visit in their payload — production
/// always emits via `Emitter::emit_scoped`.
async fn append_scoped_stage_event(
state: &Arc<AppState>,
run_id: RunId,
node_id: &str,
visit: u32,
event: &workflow_event::Event,
) {
let scope = fabro_workflow::event::StageScope {
node_id: node_id.to_string(),
visit,
parallel_group_id: None,
parallel_branch_id: None,
};
let stored = fabro_workflow::event::to_run_event_at(&run_id, event, Utc::now(), Some(&scope));
let payload = fabro_workflow::event::build_redacted_event_payload(&stored, &run_id).unwrap();
let run_store = state.store.open_run(&run_id).await.unwrap();
run_store.append_event(&payload).await.unwrap();
}
fn stage_status<'a>(body: &'a serde_json::Value, id: &str) -> &'a str {
body["data"]
.as_array()
@ -2134,7 +2158,58 @@ async fn list_run_stages_projects_retrying_until_completion() {
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
workflow_event::Event::StageStarted {
])
.await;
append_scoped_stage_event(
&state,
run_id,
"setup",
1,
&workflow_event::Event::StageStarted {
node_id: "setup".to_string(),
name: "Setup".to_string(),
index: 0,
handler_type: "command".to_string(),
attempt: 1,
max_attempts: 1,
},
)
.await;
append_scoped_stage_event(
&state,
run_id,
"setup",
1,
&workflow_event::Event::StageCompleted {
node_id: "setup".to_string(),
name: "Setup".to_string(),
index: 0,
duration_ms: 5,
status: "succeeded".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
},
)
.await;
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageStarted {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 1,
@ -2142,16 +2217,31 @@ async fn list_run_stages_projects_retrying_until_completion() {
attempt: 1,
max_attempts: 3,
},
workflow_event::Event::StageFailed {
)
.await;
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageFailed {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 1,
failure: FailureDetail::new("try again", FailureCategory::TransientInfra),
will_retry: true,
duration_ms: 10,
billing: None,
actor: None,
},
workflow_event::Event::StageRetrying {
)
.await;
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageRetrying {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 1,
@ -2159,41 +2249,9 @@ async fn list_run_stages_projects_retrying_until_completion() {
max_attempts: 3,
delay_ms: 100,
},
])
)
.await;
let mut node_outcomes = HashMap::new();
node_outcomes.insert("setup".to_string(), Outcome::success());
let mut checkpoint = Checkpoint {
timestamp: Utc::now(),
current_node: "setup".to_string(),
completed_nodes: vec!["setup".to_string()],
node_retries: HashMap::new(),
context_values: HashMap::new(),
node_outcomes,
next_node_id: Some("work".to_string()),
git_commit_sha: None,
loop_failure_signatures: HashMap::new(),
restart_failure_signatures: HashMap::new(),
node_visits: HashMap::new(),
};
let run_dir = std::env::temp_dir().join(format!("fabro-server-test-{run_id}"));
std::fs::create_dir_all(&run_dir).unwrap();
let mut managed = managed_run(
MINIMAL_DOT.to_string(),
RunStatus::Running,
Utc::now(),
run_dir,
RunExecutionMode::Start,
);
managed.checkpoint = Some(checkpoint.clone());
state
.runs
.lock()
.expect("runs lock poisoned")
.insert(run_id, managed);
let response = app
.clone()
.oneshot(
@ -2206,29 +2264,14 @@ async fn list_run_stages_projects_retrying_until_completion() {
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(stage_status(&body, "setup"), "succeeded");
assert_eq!(stage_status(&body, "work"), "retrying");
assert_eq!(stage_status(&body, "setup@1"), "succeeded");
assert_eq!(stage_status(&body, "work@1"), "retrying");
let mut work_outcome = Outcome::success();
work_outcome.status = StageOutcome::PartiallySucceeded;
checkpoint.completed_nodes.push("work".to_string());
checkpoint
.node_outcomes
.insert("work".to_string(), work_outcome);
checkpoint.current_node = "work".to_string();
checkpoint.next_node_id = Some("exit".to_string());
state
.runs
.lock()
.expect("runs lock poisoned")
.get_mut(&run_id)
.unwrap()
.checkpoint = Some(checkpoint);
let run_store = state.store.open_run(&run_id).await.unwrap();
workflow_event::append_event(
&run_store,
&run_id,
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageCompleted {
node_id: "work".to_string(),
name: "Work".to_string(),
@ -2252,8 +2295,7 @@ async fn list_run_stages_projects_retrying_until_completion() {
max_attempts: 3,
},
)
.await
.unwrap();
.await;
let response = app
.oneshot(
@ -2266,7 +2308,574 @@ async fn list_run_stages_projects_retrying_until_completion() {
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(stage_status(&body, "work"), "partially_succeeded");
assert_eq!(stage_status(&body, "work@1"), "partially_succeeded");
}
fn stage_entry<'a>(body: &'a serde_json::Value, id: &str) -> &'a serde_json::Value {
body["data"]
.as_array()
.unwrap()
.iter()
.find(|stage| stage["id"] == id)
.unwrap_or_else(|| panic!("stage {id} not found in {body:#?}"))
}
fn test_billed_usage(
model_id: &str,
input_tokens: i64,
output_tokens: i64,
) -> fabro_model::BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[tokio::test]
async fn list_run_stages_distinguishes_visits() {
let state = test_app_state_with_isolated_storage();
let app = crate::test_support::build_test_router(Arc::clone(&state));
let run_id = RunId::new();
create_durable_run_with_events(&state, run_id, &[
workflow_event::Event::RunSubmitted {
definition_blob: None,
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
])
.await;
// First visit of `verify` — failed.
append_scoped_stage_event(
&state,
run_id,
"verify",
1,
&workflow_event::Event::StageStarted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
handler_type: "command".to_string(),
attempt: 1,
max_attempts: 1,
},
)
.await;
append_scoped_stage_event(
&state,
run_id,
"verify",
1,
&workflow_event::Event::StageCompleted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
duration_ms: 1500,
status: "failed".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
},
)
.await;
// Second visit of `verify` — running.
append_scoped_stage_event(
&state,
run_id,
"verify",
2,
&workflow_event::Event::StageStarted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
handler_type: "command".to_string(),
attempt: 1,
max_attempts: 1,
},
)
.await;
let response = app
.clone()
.oneshot(
Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/stages")))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
let data = body["data"].as_array().unwrap();
let verify_entries: Vec<_> = data.iter().filter(|s| s["node_id"] == "verify").collect();
assert_eq!(verify_entries.len(), 2, "expected two verify visits");
let first = stage_entry(&body, "verify@1");
assert_eq!(first["node_id"], "verify");
assert_eq!(first["visit"], 1);
assert_eq!(first["status"], "failed");
assert_eq!(first["duration_secs"], 1.5);
let second = stage_entry(&body, "verify@2");
assert_eq!(second["node_id"], "verify");
assert_eq!(second["visit"], 2);
assert_eq!(second["status"], "running");
// Old `dot_id` field must be gone.
assert!(first.get("dot_id").is_none(), "dot_id should be removed");
}
/// `checkpoint.completed_nodes` records every visit, so a looped node appears
/// once per re-entry. Billing must dedup so a retried node renders as one row
/// and `runtime_secs` is summed across all visits exactly once.
#[tokio::test]
async fn run_billing_dedups_retried_nodes_and_sums_their_durations() {
let state = test_app_state_with_isolated_storage();
let app = crate::test_support::build_test_router(Arc::clone(&state));
let run_id = RunId::new();
create_durable_run_with_events(&state, run_id, &[
workflow_event::Event::RunSubmitted {
definition_blob: None,
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
])
.await;
// Visit 1 of `verify` — completed in 1.5s.
append_scoped_stage_event(
&state,
run_id,
"verify",
1,
&workflow_event::Event::StageCompleted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
duration_ms: 1500,
status: "failed".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
},
)
.await;
// Visit 2 of `verify` — completed in 0.8s.
append_scoped_stage_event(
&state,
run_id,
"verify",
2,
&workflow_event::Event::StageCompleted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
duration_ms: 800,
status: "succeeded".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
},
)
.await;
// Checkpoint records `verify` twice (once per visit) — this is what makes
// the dedup necessary.
let run_store = state.store.open_run(&run_id).await.unwrap();
workflow_event::append_event(
&run_store,
&run_id,
&workflow_event::Event::CheckpointCompleted {
node_id: "verify".to_string(),
status: "running".to_string(),
current_node: "verify".to_string(),
completed_nodes: vec!["verify".to_string(), "verify".to_string()],
node_retries: std::collections::BTreeMap::new(),
context_values: std::collections::BTreeMap::new(),
node_outcomes: std::collections::BTreeMap::from([(
"verify".to_string(),
Outcome::default(),
)]),
next_node_id: Some("done".to_string()),
git_commit_sha: None,
loop_failure_signatures: std::collections::BTreeMap::new(),
restart_failure_signatures: std::collections::BTreeMap::new(),
node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]),
diff: None,
},
)
.await
.unwrap();
let response = app
.clone()
.oneshot(
Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/billing")))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
let stages = body["stages"].as_array().unwrap();
assert_eq!(
stages.len(),
1,
"expected one row for the retried verify node"
);
assert_eq!(stages[0]["stage"]["id"], "verify");
// Duration on the row is the sum across visits (1.5s + 0.8s = 2.3s).
assert!(
(stages[0]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON,
"row runtime_secs should sum visits, got {}",
stages[0]["runtime_secs"]
);
// Totals must not double-count: a single 2.3s, not 4.6s.
assert!(
(body["totals"]["runtime_secs"].as_f64().unwrap() - 2.3).abs() < f64::EPSILON,
"totals.runtime_secs should sum visits exactly once, got {}",
body["totals"]["runtime_secs"]
);
}
#[tokio::test]
async fn run_billing_sums_usage_across_retry_visits_and_uses_latest_model() {
let state = test_app_state_with_isolated_storage();
let app = crate::test_support::build_test_router(Arc::clone(&state));
let run_id = RunId::new();
let failed_usage = test_billed_usage("gpt-old", 100, 10);
let success_usage = test_billed_usage("gpt-new", 200, 20);
create_durable_run_with_events(&state, run_id, &[
workflow_event::Event::RunSubmitted {
definition_blob: None,
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
])
.await;
append_scoped_stage_event(
&state,
run_id,
"verify",
1,
&workflow_event::Event::StageFailed {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
failure: FailureDetail::new("try again", FailureCategory::TransientInfra),
will_retry: true,
duration_ms: 1200,
billing: Some(failed_usage),
actor: None,
},
)
.await;
append_scoped_stage_event(
&state,
run_id,
"verify",
2,
&workflow_event::Event::StageCompleted {
node_id: "verify".to_string(),
name: "Verify".to_string(),
index: 1,
duration_ms: 800,
status: "succeeded".to_string(),
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(success_usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 2,
max_attempts: 2,
},
)
.await;
let mut latest_outcome: Outcome<Option<fabro_model::BilledModelUsage>> = Outcome::success();
latest_outcome.usage = Some(success_usage);
latest_outcome.duration_ms = Some(800);
let run_store = state.store.open_run(&run_id).await.unwrap();
workflow_event::append_event(
&run_store,
&run_id,
&workflow_event::Event::CheckpointCompleted {
node_id: "verify".to_string(),
status: "running".to_string(),
current_node: "verify".to_string(),
completed_nodes: vec!["verify".to_string(), "verify".to_string()],
node_retries: std::collections::BTreeMap::from([("verify".to_string(), 2)]),
context_values: std::collections::BTreeMap::new(),
node_outcomes: std::collections::BTreeMap::from([(
"verify".to_string(),
latest_outcome,
)]),
next_node_id: None,
git_commit_sha: None,
loop_failure_signatures: std::collections::BTreeMap::new(),
restart_failure_signatures: std::collections::BTreeMap::new(),
node_visits: std::collections::BTreeMap::from([("verify".to_string(), 2usize)]),
diff: None,
},
)
.await
.unwrap();
let response = app
.clone()
.oneshot(
Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/billing")))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
let stages = body["stages"].as_array().unwrap();
assert_eq!(stages.len(), 1);
assert_eq!(stages[0]["stage"]["id"], "verify");
assert_eq!(stages[0]["model"]["id"], "gpt-new");
assert_eq!(stages[0]["billing"]["input_tokens"], 300);
assert_eq!(stages[0]["billing"]["output_tokens"], 30);
assert_eq!(stages[0]["billing"]["total_usd_micros"], 330);
assert!((stages[0]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON);
assert_eq!(body["totals"]["input_tokens"], 300);
assert_eq!(body["totals"]["output_tokens"], 30);
assert_eq!(body["totals"]["total_usd_micros"], 330);
assert!((body["totals"]["runtime_secs"].as_f64().unwrap() - 2.0).abs() < f64::EPSILON);
let by_model = body["by_model"].as_array().unwrap();
assert_eq!(by_model.len(), 2);
let old_model = by_model
.iter()
.find(|entry| entry["model"]["id"] == "gpt-old")
.unwrap();
let new_model = by_model
.iter()
.find(|entry| entry["model"]["id"] == "gpt-new")
.unwrap();
assert_eq!(old_model["stages"], 1);
assert_eq!(old_model["billing"]["input_tokens"], 100);
assert_eq!(new_model["stages"], 1);
assert_eq!(new_model["billing"]["input_tokens"], 200);
}
#[tokio::test]
async fn list_run_stages_shows_retrying_after_failed_event() {
let state = test_app_state_with_isolated_storage();
let app = crate::test_support::build_test_router(Arc::clone(&state));
let run_id = RunId::new();
create_durable_run_with_events(&state, run_id, &[
workflow_event::Event::RunSubmitted {
definition_blob: None,
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
])
.await;
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageStarted {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 0,
handler_type: "command".to_string(),
attempt: 1,
max_attempts: 3,
},
)
.await;
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageFailed {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 0,
failure: FailureDetail::new("flake", FailureCategory::TransientInfra),
will_retry: true,
duration_ms: 5,
billing: None,
actor: None,
},
)
.await;
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageRetrying {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 0,
attempt: 2,
max_attempts: 3,
delay_ms: 50,
},
)
.await;
let response = app
.clone()
.oneshot(
Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/stages")))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(stage_status(&body, "work@1"), "retrying");
}
#[tokio::test]
async fn list_run_stages_shows_retrying_when_failed_will_retry() {
let state = test_app_state_with_isolated_storage();
let app = crate::test_support::build_test_router(Arc::clone(&state));
let run_id = RunId::new();
create_durable_run_with_events(&state, run_id, &[
workflow_event::Event::RunSubmitted {
definition_blob: None,
},
workflow_event::Event::RunStarting,
workflow_event::Event::RunRunning,
])
.await;
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageStarted {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 0,
handler_type: "command".to_string(),
attempt: 1,
max_attempts: 3,
},
)
.await;
// Only StageFailed, no StageRetrying yet — should still render retrying
// because props.will_retry is true.
append_scoped_stage_event(
&state,
run_id,
"work",
1,
&workflow_event::Event::StageFailed {
node_id: "work".to_string(),
name: "Work".to_string(),
index: 0,
failure: FailureDetail::new("flake", FailureCategory::TransientInfra),
will_retry: true,
duration_ms: 5,
billing: None,
actor: None,
},
)
.await;
let response = app
.clone()
.oneshot(
Request::builder()
.method("GET")
.uri(api(&format!("/runs/{run_id}/stages")))
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let body = response_json!(response, StatusCode::OK).await;
assert_eq!(stage_status(&body, "work@1"), "retrying");
}
async fn append_raw_run_event(
@ -3665,7 +4274,13 @@ async fn create_run_pull_request_creates_and_persists_record() {
.header("authorization", "Bearer openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload("Narrative from mock."));
.json_body(openai_responses_payload(
&serde_json::to_string(&json!({
"title": "Mock title",
"body": "Narrative from mock.",
}))
.unwrap(),
));
})
.await;
let openai_base_url = llm.url("/v1");
@ -5829,6 +6444,62 @@ async fn get_aggregate_billing_returns_zeros_initially() {
assert!(body["by_model"].as_array().unwrap().is_empty());
}
#[test]
fn aggregate_billing_counts_projection_rollup_usage_visits() {
let mut accumulator = BillingAccumulator::default();
let rollup = fabro_workflow::ProjectionBillingRollup {
stages: Vec::new(),
totals: BilledTokenCounts {
input_tokens: 300,
output_tokens: 30,
total_tokens: 330,
reasoning_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
total_usd_micros: Some(330),
},
by_model: vec![
fabro_workflow::ProjectionBillingByModel {
model_id: "gpt-old".to_string(),
stages: 1,
billing: BilledTokenCounts {
input_tokens: 100,
output_tokens: 10,
total_tokens: 110,
reasoning_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
total_usd_micros: Some(110),
},
},
fabro_workflow::ProjectionBillingByModel {
model_id: "gpt-new".to_string(),
stages: 1,
billing: BilledTokenCounts {
input_tokens: 200,
output_tokens: 20,
total_tokens: 220,
reasoning_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
total_usd_micros: Some(220),
},
},
],
runtime_ms: 2000,
billed_visit_count: 2,
};
accumulate_billing_rollup(&mut accumulator, &rollup);
assert_eq!(accumulator.total_runs, 1);
assert_eq!(accumulator.total_runtime_secs, 2.0);
assert_eq!(accumulator.by_model["gpt-old"].stages, 1);
assert_eq!(accumulator.by_model["gpt-old"].billing.input_tokens, 100);
assert_eq!(accumulator.by_model["gpt-new"].stages, 1);
assert_eq!(accumulator.by_model["gpt-new"].billing.input_tokens, 200);
}
#[tokio::test]
async fn post_runs_returns_submitted_status() {
let state = test_app_state();

6
lib/crates/fabro-server/tests/it/event_pagination.rs
View file

@ -31,7 +31,7 @@ async fn get_json(app: &axum::Router, uri: &str) -> serde_json::Value {
async fn demo_stage_events_default_returns_all_fixture_events_with_no_more() {
let app = fabro_server::test_support::build_test_router(test_app_state());
let body = get_json(&app, "/api/v1/runs/run-1/stages/detect-drift/events").await;
let body = get_json(&app, "/api/v1/runs/run-1/stages/detect-drift@1/events").await;
let data = body["data"].as_array().expect("data is an array");
assert_eq!(data.len(), 7, "all seven fixture events should be returned");
@ -44,7 +44,7 @@ async fn demo_stage_events_limit_one_signals_has_more() {
let body = get_json(
&app,
"/api/v1/runs/run-1/stages/detect-drift/events?limit=1",
"/api/v1/runs/run-1/stages/detect-drift@1/events?limit=1",
)
.await;
let data = body["data"].as_array().expect("data is an array");
@ -60,7 +60,7 @@ async fn demo_stage_events_since_seq_filters_out_earlier_events() {
// The fixture seqs are 1..=7. since_seq=4 should skip the first three.
let body = get_json(
&app,
"/api/v1/runs/run-1/stages/detect-drift/events?since_seq=4",
"/api/v1/runs/run-1/stages/detect-drift@1/events?since_seq=4",
)
.await;
let data = body["data"].as_array().expect("data is an array");

7
lib/crates/fabro-store/src/artifact_store.rs
View file

@ -297,8 +297,13 @@ fn decode_artifact_location(
))
})?;
let (retry, filename) = decode_retry_and_filename(location, &mut parts)?;
let stage_id = StageId::try_new(node_id, visit).map_err(|err| {
Error::Other(format!(
"artifact location {location} has an invalid stage id: {err}"
))
})?;
Ok(NodeArtifact {
node: StageId::new(node_id, visit),
node: stage_id,
retry,
filename,
size,

327
lib/crates/fabro-store/src/run_state.rs
View file

@ -9,8 +9,8 @@ use fabro_types::run_event::{
use fabro_types::{
BilledModelUsage, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord,
Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId,
RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageOutcome,
StageProjection, StartRecord, TerminalStatus, first_event_seq,
RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageId,
StageOutcome, StageProjection, StartRecord, TerminalStatus, first_event_seq,
};
use fabro_util::error::render_with_causes;
use serde_json::Value;
@ -297,52 +297,59 @@ impl RunProjectionReducer for RunProjection {
);
}
EventBody::StagePrompt(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.prompt = Some(props.text.clone());
stage.provider_used = provider_used_from_prompt(props);
}
EventBody::PromptCompleted(props) => {
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.response = Some(props.response.clone());
}
EventBody::StageCompleted(props) => {
let Some(node_id) = stored.node_id.as_deref() else {
return Ok(());
};
let visit = stage_visit(node_id, props.node_visits.as_ref(), self).unwrap_or(1);
let response = props.response.clone();
let outcome = stage_outcome_from_props(props);
let completion = stage_completion_from_outcome(&outcome, ts);
let stage = self.stage_entry(node_id, visit, first_event_seq(event.seq));
let Some(stage) =
stage_at_completed_visit(self, stored, props.node_visits.as_ref(), event.seq)
else {
return Ok(());
};
stage.response = response;
stage.completion = Some(completion);
stage.duration_ms = Some(props.duration_ms);
stage.usage.clone_from(&props.billing);
}
EventBody::StageFailed(props) => {
let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone());
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: false,
retry_requested: props.will_retry,
},
notes: None,
failure_reason,
timestamp: ts,
});
stage.duration_ms = Some(props.duration_ms);
stage.usage.clone_from(&props.billing);
}
EventBody::AgentSessionStarted(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.provider_used = Some(provider_used_from_agent_session_started(props));
}
EventBody::AgentCliStarted(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.provider_used = Some(provider_used_from_agent_cli_started(props));
@ -351,7 +358,7 @@ impl RunProjectionReducer for RunProjection {
let script_invocation = serde_json::to_value(props).map_err(|err| {
Error::InvalidEvent(format!("invalid command.started payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.script_invocation = Some(script_invocation);
@ -360,7 +367,7 @@ impl RunProjectionReducer for RunProjection {
let script_timing = serde_json::to_value(props).map_err(|err| {
Error::InvalidEvent(format!("invalid command.completed payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.stdout = Some(props.stdout.clone());
@ -376,7 +383,7 @@ impl RunProjectionReducer for RunProjection {
let parallel_results = serde_json::to_value(&props.results).map_err(|err| {
Error::InvalidEvent(format!("invalid parallel.completed payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.parallel_results = Some(parallel_results);
@ -394,6 +401,9 @@ fn stage_at_visit<'a>(
visit: u32,
seq: u32,
) -> Option<&'a mut StageProjection> {
if visit == 0 {
return None;
}
let node_id = stored.node_id.as_deref()?;
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
@ -408,6 +418,51 @@ fn stage_at_current_visit<'a>(
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
fn stage_at_stored_stage_id<'a>(
state: &'a mut RunProjection,
stage_id: &StageId,
seq: u32,
) -> &'a mut StageProjection {
state.stage_entry(stage_id.node_id(), stage_id.visit(), first_event_seq(seq))
}
fn stage_at_stored_or_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
visit: u32,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
stage_at_visit(state, stored, visit, seq)
}
fn stage_at_stored_or_current_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
stage_at_current_visit(state, stored, seq)
}
fn stage_at_completed_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
node_visits: Option<&BTreeMap<String, usize>>,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
let node_id = stored.node_id.as_deref()?;
let visit = stage_visit(node_id, node_visits, state).unwrap_or(1);
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> RunSummary {
let workflow_name = state.spec.as_ref().map(|spec| {
if spec.graph.name.is_empty() {
@ -529,6 +584,7 @@ fn stage_visit(
node_visits
.and_then(|visits| visits.get(node_id).copied())
.and_then(|visit| u32::try_from(visit).ok())
.filter(|visit| *visit > 0)
.or_else(|| state.current_visit_for(node_id))
}
@ -613,12 +669,13 @@ mod tests {
use fabro_types::run_event::run::RunFailedProps;
use fabro_types::run_event::{
CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps,
RunControlEffectProps, StagePromptProps, StageStartedProps,
RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps,
StageStartedProps,
};
use fabro_types::{
BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, QuestionType, RunBlobId,
RunControlAction, RunEvent, RunStatus, StageOutcome, SuccessReason, TerminalStatus,
WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, BlockedReason, Checkpoint, EventBody, FailureReason, Outcome,
QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, StageOutcome,
SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
@ -651,6 +708,32 @@ mod tests {
event
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
fn usage_json(usage: &BilledModelUsage) -> serde_json::Value {
serde_json::to_value(usage).unwrap()
}
fn test_raw_event(
seq: u32,
event: &str,
@ -861,6 +944,210 @@ mod tests {
assert_eq!(stage.prompt.as_deref(), Some("prompt"));
}
#[test]
fn stage_completed_event_captures_duration_and_usage_per_visit() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 123, 45);
state
.apply_event(&test_event(
3,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms: 789,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("done".to_string()),
attempt: 1,
max_attempts: 1,
}),
Some("build"),
))
.unwrap();
let stage = state.stage(&StageId::new("build", 1)).unwrap();
assert_eq!(stage.duration_ms, Some(789));
assert_eq!(stage.usage.as_ref(), Some(&usage));
}
#[test]
fn stage_failed_event_captures_duration_and_usage_per_visit() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
let usage = test_usage("gpt-5.2", 321, 54);
state
.apply_event(&test_stage_event(
2,
EventBody::StageStarted(StageStartedProps {
index: 0,
handler_type: "agent".to_string(),
attempt: 1,
max_attempts: 1,
}),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_raw_event(
3,
"stage.failed",
&json!({
"index": 0,
"failure": {
"message": "provider failed",
"failure_class": "transient_infra"
},
"will_retry": false,
"duration_ms": 654,
"billing": usage_json(&usage)
}),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(654));
assert_eq!(stage.usage.as_ref(), Some(&usage));
}
#[test]
fn two_visits_of_one_node_retain_distinct_usage() {
let mut state = RunProjection::default();
let first_usage = test_usage("gpt-5.2", 100, 10);
let second_usage = test_usage("gpt-5.2", 200, 20);
for (seq, visit, duration_ms, usage) in [
(3, 1usize, 111, first_usage.clone()),
(4, 2usize, 222, second_usage.clone()),
] {
state
.apply_event(&test_event(
seq,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: Some(BTreeMap::from([("build".to_string(), visit)])),
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
}),
Some("build"),
))
.unwrap();
}
let first_stage = state.stage(&StageId::new("build", 1)).unwrap();
let second_stage = state.stage(&StageId::new("build", 2)).unwrap();
assert_eq!(first_stage.duration_ms, Some(111));
assert_eq!(first_stage.usage.as_ref(), Some(&first_usage));
assert_eq!(second_stage.duration_ms, Some(222));
assert_eq!(second_stage.usage.as_ref(), Some(&second_usage));
}
#[test]
fn stage_completed_prefers_stored_stage_id_over_legacy_node_visits() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 300, 30);
let scoped_stage_id = StageId::new("build", 2);
state
.apply_event(&test_stage_event(
3,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms: 333,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: Some(BTreeMap::from([("build".to_string(), 1usize)])),
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("done".to_string()),
attempt: 1,
max_attempts: 1,
}),
scoped_stage_id.clone(),
))
.unwrap();
assert!(
state.stage(&StageId::new("build", 1)).is_none(),
"legacy node_visits must not override stored stage_id"
);
let stage = state.stage(&scoped_stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(333));
assert_eq!(stage.usage.as_ref(), Some(&usage));
assert_eq!(stage.response.as_deref(), Some("done"));
}
#[test]
fn stage_failed_prefers_stored_stage_id_and_preserves_retry_request() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 400, 40);
let scoped_stage_id = StageId::new("build", 2);
state
.apply_event(&test_stage_event(
3,
EventBody::StageFailed(StageFailedProps {
index: 0,
failure: Some(fabro_types::FailureDetail::new(
"try again",
fabro_types::FailureCategory::TransientInfra,
)),
will_retry: true,
duration_ms: 444,
billing: Some(usage.clone()),
}),
scoped_stage_id.clone(),
))
.unwrap();
assert!(
state.stage(&StageId::new("build", 1)).is_none(),
"current-visit fallback must not override stored stage_id"
);
let stage = state.stage(&scoped_stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(444));
assert_eq!(stage.usage.as_ref(), Some(&usage));
let completion = stage.completion.as_ref().unwrap();
assert_eq!(completion.outcome, StageOutcome::Failed {
retry_requested: true,
});
assert_eq!(completion.failure_reason.as_deref(), Some("try again"));
}
#[test]
fn checkpoint_completed_creates_projection_entry_for_skipped_stage() {
let mut state = RunProjection::default();

106
lib/crates/fabro-store/src/slate/run_store.rs
View file

@ -12,7 +12,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream;
use super::blob_store::BlobStore;
use crate::run_state::{EventProjectionCache, RunProjectionReducer, build_summary};
use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, keys};
use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, StageId, keys};
const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024;
#[derive(Clone)]
@ -213,24 +213,24 @@ impl RunDatabase {
list_events_from_with_limit(&self.inner.db, &self.inner.run_id, start_seq, limit).await
}
/// Returns up to `limit + 1` events for the given workflow node,
/// Returns up to `limit + 1` events for the given stage visit,
/// starting at `start_seq`. The `+1` lets callers compute `has_more`.
///
/// Implementation note: scans the unbounded run-event prefix and
/// filters by `node_id` *before* applying `limit`, so a stage with
/// filters by stage identity *before* applying `limit`, so a stage with
/// matches sparsely scattered late in the event log still returns its
/// full slice (no premature truncation from a generic `limit`-bounded
/// scan).
pub async fn list_events_for_node_from_with_limit(
pub async fn list_events_for_stage_from_with_limit(
&self,
node_id: &str,
stage_id: &StageId,
start_seq: u32,
limit: usize,
) -> Result<Vec<EventEnvelope>> {
list_events_for_node_from_with_limit(
list_events_for_stage_from_with_limit(
&self.inner.db,
&self.inner.run_id,
node_id,
stage_id,
start_seq,
limit,
)
@ -372,30 +372,33 @@ where
Ok(events)
}
async fn list_events_for_node_from_with_limit<R>(
async fn list_events_for_stage_from_with_limit<R>(
db: &R,
run_id: &RunId,
node_id: &str,
stage_id: &StageId,
start_seq: u32,
limit: usize,
) -> Result<Vec<EventEnvelope>>
where
R: DbRead + Sync,
{
// Unbounded scan first: filtering by node_id with a generic
// Unbounded scan first: filtering by stage identity with a generic
// limit-bounded scan would silently drop matches whenever the stage's
// events are sparse late in the event log.
//
// We probe just the `node_id` field with a small partial deserialize and
// We probe just the stage identity fields with a small partial deserialize and
// only run the full `RunEvent` parse on matches. Most events in a run
// belong to other nodes, so this avoids deserializing large payloads
// (`agent.tool.completed.output`, `agent.message.text`, …) we'd discard.
#[derive(serde::Deserialize)]
struct NodeIdProbe<'a> {
struct StageIdProbe<'a> {
#[serde(default, borrow)]
node_id: Option<&'a str>,
stage_id: Option<&'a str>,
#[serde(default, borrow)]
node_id: Option<&'a str>,
}
let stage_id_string = stage_id.to_string();
let max_events = limit.saturating_add(1);
let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?;
let mut events: Vec<EventEnvelope> = Vec::new();
@ -407,8 +410,12 @@ where
if seq < start_seq {
continue;
}
let probe: NodeIdProbe = serde_json::from_slice(&entry.value)?;
if probe.node_id != Some(node_id) {
let probe: StageIdProbe = serde_json::from_slice(&entry.value)?;
let matches_stage_id = probe.stage_id == Some(stage_id_string.as_str());
let matches_legacy_node_id = probe.stage_id.is_none()
&& stage_id.visit() == 1
&& probe.node_id == Some(stage_id.node_id());
if !matches_stage_id && !matches_legacy_node_id {
continue;
}
let event: RunEvent = serde_json::from_slice(&entry.value)?;
@ -460,7 +467,7 @@ mod tests {
use std::sync::Arc;
use std::time::Duration;
use fabro_types::RunId;
use fabro_types::{RunId, StageId};
use object_store::memory::InMemory;
use serde_json::json;
@ -484,6 +491,15 @@ mod tests {
}
fn stage_prompt_payload(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload {
stage_prompt_payload_for_stage(run_id, idx, node_id, None)
}
fn stage_prompt_payload_for_stage(
run_id: &RunId,
idx: u32,
node_id: Option<&str>,
stage_id: Option<&StageId>,
) -> EventPayload {
let mut value = json!({
"id": format!("evt-{idx}"),
"ts": "2026-04-09T12:00:00Z",
@ -500,6 +516,12 @@ mod tests {
.unwrap()
.insert("node_id".into(), json!(node_id));
}
if let Some(stage_id) = stage_id {
value
.as_object_mut()
.unwrap()
.insert("stage_id".into(), json!(stage_id.to_string()));
}
EventPayload::new(value, run_id).unwrap()
}
@ -511,7 +533,7 @@ mod tests {
}
#[tokio::test]
async fn list_events_for_node_returns_only_matching_events_in_seq_order() {
async fn list_events_for_stage_returns_only_matching_events_in_seq_order() {
let run = fresh_run().await;
let run_id = run.run_id();
run.append_event(&stage_prompt_payload(&run_id, 1, Some("alpha")))
@ -525,7 +547,7 @@ mod tests {
.unwrap();
let events = run
.list_events_for_node_from_with_limit("alpha", 1, 100)
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100)
.await
.unwrap();
@ -534,7 +556,7 @@ mod tests {
}
#[tokio::test]
async fn list_events_for_node_skips_events_with_no_node_id() {
async fn list_events_for_stage_skips_events_with_no_stage_identity() {
let run = fresh_run().await;
let run_id = run.run_id();
run.append_event(&stage_prompt_payload(&run_id, 1, None))
@ -545,7 +567,7 @@ mod tests {
.unwrap();
let events = run
.list_events_for_node_from_with_limit("alpha", 1, 100)
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100)
.await
.unwrap();
@ -554,7 +576,7 @@ mod tests {
}
#[tokio::test]
async fn list_events_for_node_paginates_via_start_seq_on_filtered_slice() {
async fn list_events_for_stage_paginates_via_start_seq_on_filtered_slice() {
let run = fresh_run().await;
let run_id = run.run_id();
for idx in 1..=5 {
@ -566,7 +588,7 @@ mod tests {
// alpha events live at seqs 1, 3, 5. Start at seq=2 should skip seq=1.
let events = run
.list_events_for_node_from_with_limit("alpha", 2, 100)
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 2, 100)
.await
.unwrap();
@ -575,7 +597,7 @@ mod tests {
}
#[tokio::test]
async fn list_events_for_node_walks_past_unrelated_events_for_sparse_matches() {
async fn list_events_for_stage_walks_past_unrelated_events_for_sparse_matches() {
let run = fresh_run().await;
let run_id = run.run_id();
// 200 unrelated events first.
@ -594,7 +616,7 @@ mod tests {
// limit smaller than the number of unrelated events would have
// truncated the upstream scan if we had post-filtered.
let events = run
.list_events_for_node_from_with_limit("alpha", 1, 5)
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 5)
.await
.unwrap();
@ -603,7 +625,7 @@ mod tests {
}
#[tokio::test]
async fn list_events_for_node_returns_limit_plus_one_for_has_more_signal() {
async fn list_events_for_stage_returns_limit_plus_one_for_has_more_signal() {
let run = fresh_run().await;
let run_id = run.run_id();
for idx in 1..=5 {
@ -613,7 +635,7 @@ mod tests {
}
let events = run
.list_events_for_node_from_with_limit("alpha", 1, 2)
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 2)
.await
.unwrap();
@ -621,4 +643,36 @@ mod tests {
// caller can compute has_more.
assert_eq!(events.len(), 3);
}
#[tokio::test]
async fn list_events_for_stage_prefers_stage_id_over_node_id() {
let run = fresh_run().await;
let run_id = run.run_id();
let first_visit = StageId::new("verify", 1);
let second_visit = StageId::new("verify", 2);
run.append_event(&stage_prompt_payload_for_stage(
&run_id,
1,
Some("verify"),
Some(&first_visit),
))
.await
.unwrap();
run.append_event(&stage_prompt_payload_for_stage(
&run_id,
2,
Some("verify"),
Some(&second_visit),
))
.await
.unwrap();
let events = run
.list_events_for_stage_from_with_limit(&second_visit, 1, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![2]);
}
}

30
lib/crates/fabro-store/tests/serializable_projection.rs
View file

@ -5,8 +5,8 @@ use fabro_store::{RunProjection, SerializableProjection, StageId};
use fabro_types::graph::Graph;
use fabro_types::run::RunSpec;
use fabro_types::{
Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, StartRecord,
TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome,
StartRecord, TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
@ -52,6 +52,28 @@ fn sample_checkpoint() -> Checkpoint {
}
}
fn sample_usage() -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": "gpt-5.2"
},
"tokens": {
"input_tokens": 123,
"output_tokens": 45
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": 168
}))
.expect("sample usage should deserialize")
}
#[test]
fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
let stage_id = StageId::new("build", 2);
@ -94,6 +116,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
stage.script_invocation = Some(json!({ "command": "cargo test" }));
stage.script_timing = Some(json!({ "duration_ms": 10 }));
stage.parallel_results = Some(json!([{ "stage": "fanout@1" }]));
stage.duration_ms = Some(1234);
stage.usage = Some(sample_usage());
stage.stdout = Some("stdout".to_string());
stage.stderr = Some("stderr".to_string());
@ -138,6 +162,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
node.parallel_results,
Some(json!([{ "stage": "fanout@1" }]))
);
assert_eq!(node.duration_ms, Some(1234));
assert_eq!(node.usage, Some(sample_usage()));
}
#[test]

2
lib/crates/fabro-types/src/lib.rs
View file

@ -80,7 +80,7 @@ pub use run_summary::RunSummary;
pub use sandbox_record::SandboxRecord;
pub use secret::{SecretMetadata, SecretType};
pub use stage_completion::StageCompletion;
pub use stage_id::{ParallelBranchId, StageId};
pub use stage_id::{InvalidStageVisit, ParallelBranchId, StageId};
pub use start::StartRecord;
pub use status::{
BlockedReason, FailureReason, InvalidTransition, ParseFailureReasonError,

25
lib/crates/fabro-types/src/outcome.rs
View file

@ -138,12 +138,32 @@ impl From<StageOutcome> for StageState {
match outcome {
StageOutcome::Succeeded => Self::Succeeded,
StageOutcome::PartiallySucceeded => Self::PartiallySucceeded,
StageOutcome::Failed { .. } => Self::Failed,
StageOutcome::Failed {
retry_requested: true,
} => Self::Retrying,
StageOutcome::Failed {
retry_requested: false,
} => Self::Failed,
StageOutcome::Skipped => Self::Skipped,
}
}
}
#[cfg(test)]
mod stage_state_tests {
use super::{StageOutcome, StageState};
#[test]
fn retry_requested_failure_projects_as_retrying() {
assert_eq!(
StageState::from(StageOutcome::Failed {
retry_requested: true,
}),
StageState::Retrying
);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FailureCategory {
@ -340,9 +360,10 @@ mod tests {
StageState::from(StageOutcome::Failed {
retry_requested: true,
}),
StageState::Failed
StageState::Retrying
);
assert!(StageState::Cancelled.is_terminal());
assert!(!StageState::Retrying.is_terminal());
assert!(!StageState::Running.is_terminal());
}
}

2
lib/crates/fabro-types/src/run_event/stage.rs
View file

@ -57,6 +57,8 @@ pub struct StageFailedProps {
pub will_retry: bool,
#[serde(default)]
pub duration_ms: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub billing: Option<BilledModelUsage>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

123
lib/crates/fabro-types/src/run_projection.rs
View file

@ -4,9 +4,9 @@ use std::num::NonZeroU32;
use chrono::{DateTime, Utc};
use crate::{
Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro,
RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, StageCompletion, StageId,
StartRecord,
BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition,
PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord,
StageCompletion, StageId, StartRecord,
};
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
@ -44,6 +44,10 @@ pub struct StageProjection {
pub prompt: Option<String>,
pub response: Option<String>,
pub completion: Option<StageCompletion>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage: Option<BilledModelUsage>,
pub provider_used: Option<serde_json::Value>,
pub diff: Option<String>,
pub script_invocation: Option<serde_json::Value>,
@ -78,6 +82,8 @@ impl StageProjection {
prompt: None,
response: None,
completion: None,
duration_ms: None,
usage: None,
provider_used: None,
diff: None,
script_invocation: None,
@ -99,12 +105,32 @@ impl RunProjection {
self.stages.get(stage)
}
/// Iterate stages in `first_event_seq` order (the chronological order in
/// which each stage's first lifecycle event was recorded). Internal
/// storage is a `HashMap`, so iteration would otherwise be
/// non-deterministic; every caller wants chronological order, so we sort
/// here once instead of asking each caller to remember.
pub fn iter_stages(&self) -> impl Iterator<Item = (&StageId, &StageProjection)> {
self.stages.iter()
let mut entries: Vec<(&StageId, &StageProjection)> = self.stages.iter().collect();
entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| {
left_stage
.first_event_seq
.cmp(&right_stage.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
entries.into_iter()
}
/// Mutable counterpart of [`iter_stages`]. Same chronological ordering.
pub fn iter_stages_mut(&mut self) -> impl Iterator<Item = (&StageId, &mut StageProjection)> {
self.stages.iter_mut()
let mut entries: Vec<(&StageId, &mut StageProjection)> = self.stages.iter_mut().collect();
entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| {
left_stage
.first_event_seq
.cmp(&right_stage.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
entries.into_iter()
}
pub fn is_empty(&self) -> bool {
@ -186,3 +212,90 @@ impl RunProjection {
}
}
}
#[cfg(test)]
mod iter_stages_tests {
use std::num::NonZeroU32;
use super::RunProjection;
fn seq(n: u32) -> NonZeroU32 {
NonZeroU32::new(n).unwrap()
}
#[test]
fn iter_stages_yields_chronological_order_across_nodes() {
let mut p = RunProjection::default();
// Insert in non-monotonic seq order to exercise the sort.
p.stage_entry("c", 1, seq(30));
p.stage_entry("a", 1, seq(10));
p.stage_entry("b", 1, seq(20));
let order: Vec<&str> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.node_id())
.collect();
assert_eq!(order, vec!["a", "b", "c"]);
}
#[test]
fn iter_stages_orders_visits_within_a_node() {
let mut p = RunProjection::default();
// Visit 2 inserted first; visit 1's earlier first_event_seq must still
// win the chronological ordering.
p.stage_entry("verify", 2, seq(50));
p.stage_entry("verify", 1, seq(20));
let visits: Vec<u32> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.visit())
.collect();
assert_eq!(visits, vec![1, 2]);
}
#[test]
fn iter_stages_mut_yields_chronological_order() {
let mut p = RunProjection::default();
p.stage_entry("c", 1, seq(30));
p.stage_entry("a", 1, seq(10));
p.stage_entry("b", 1, seq(20));
let order: Vec<String> = p
.iter_stages_mut()
.map(|(stage_id, _)| stage_id.node_id().to_string())
.collect();
assert_eq!(order, vec!["a", "b", "c"]);
}
#[test]
fn iter_stages_tie_breaks_same_first_event_seq_by_stage_id() {
for _ in 0..128 {
let mut p = RunProjection::default();
p.stage_entry("verify", 2, seq(10));
p.stage_entry("build", 1, seq(10));
p.stage_entry("verify", 1, seq(10));
let order: Vec<String> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.to_string())
.collect();
assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]);
}
}
#[test]
fn iter_stages_mut_tie_breaks_same_first_event_seq_by_stage_id() {
for _ in 0..128 {
let mut p = RunProjection::default();
p.stage_entry("verify", 2, seq(10));
p.stage_entry("build", 1, seq(10));
p.stage_entry("verify", 1, seq(10));
let order: Vec<String> = p
.iter_stages_mut()
.map(|(stage_id, _)| stage_id.to_string())
.collect();
assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]);
}
}
}

39
lib/crates/fabro-types/src/stage_id.rs
View file

@ -1,4 +1,5 @@
use std::fmt;
use std::num::NonZeroU32;
use std::str::FromStr;
use serde::de::Error as _;
@ -7,16 +8,21 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct StageId {
node_id: String,
visit: u32,
visit: NonZeroU32,
}
impl StageId {
#[must_use]
pub fn new(node_id: impl Into<String>, visit: u32) -> Self {
Self {
Self::try_new(node_id, visit).expect("stage id visit must be greater than zero")
}
pub fn try_new(node_id: impl Into<String>, visit: u32) -> Result<Self, InvalidStageVisit> {
let visit = NonZeroU32::new(visit).ok_or(InvalidStageVisit)?;
Ok(Self {
node_id: node_id.into(),
visit,
}
})
}
#[must_use]
@ -26,7 +32,7 @@ impl StageId {
#[must_use]
pub fn visit(&self) -> u32 {
self.visit
self.visit.get()
}
}
@ -47,6 +53,17 @@ impl fmt::Display for ParseStageIdError {
impl std::error::Error for ParseStageIdError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InvalidStageVisit;
impl fmt::Display for InvalidStageVisit {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("stage id visit must be greater than zero")
}
}
impl std::error::Error for InvalidStageVisit {}
impl FromStr for StageId {
type Err = ParseStageIdError;
@ -67,7 +84,7 @@ impl FromStr for StageId {
let visit = visit
.parse()
.map_err(|err| ParseStageIdError(format!("invalid stage id visit: {err}")))?;
Ok(Self::new(node_id, visit))
Self::try_new(node_id, visit).map_err(|err| ParseStageIdError(err.to_string()))
}
}
@ -224,6 +241,18 @@ mod tests {
assert!(err.to_string().starts_with("invalid stage id visit:"));
}
#[test]
fn parse_rejects_zero_visit() {
let err = "code@0".parse::<StageId>().unwrap_err();
assert_eq!(err.to_string(), "stage id visit must be greater than zero");
}
#[test]
fn try_new_rejects_zero_visit() {
let err = StageId::try_new("code", 0).unwrap_err();
assert_eq!(err.to_string(), "stage id visit must be greater than zero");
}
#[test]
fn parse_rejects_empty_node_id() {
let err = "@3".parse::<StageId>().unwrap_err();

291
lib/crates/fabro-workflow/src/billing_rollup.rs Normal file
View file

@ -0,0 +1,291 @@
use std::collections::{BTreeMap, HashMap};
use fabro_types::{BilledModelUsage, BilledTokenCounts, RunProjection};
#[derive(Debug, Clone, PartialEq)]
pub struct ProjectionBillingStage {
pub node_id: String,
pub billing: BilledTokenCounts,
pub duration_ms: u64,
pub model_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProjectionBillingByModel {
pub model_id: String,
pub stages: i64,
pub billing: BilledTokenCounts,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct ProjectionBillingRollup {
pub stages: Vec<ProjectionBillingStage>,
pub totals: BilledTokenCounts,
pub by_model: Vec<ProjectionBillingByModel>,
pub runtime_ms: u64,
pub billed_visit_count: usize,
}
impl ProjectionBillingRollup {
#[must_use]
pub fn billing_if_present(&self) -> Option<BilledTokenCounts> {
(self.billed_visit_count > 0).then(|| self.totals.clone())
}
}
#[must_use]
pub fn billing_rollup_from_projection(projection: &RunProjection) -> ProjectionBillingRollup {
let mut stage_indices = HashMap::<String, usize>::new();
let mut stages = Vec::<ProjectionBillingStage>::new();
let mut by_model = BTreeMap::<String, ProjectionBillingByModel>::new();
let mut totals = BilledTokenCounts::default();
let mut runtime_ms = 0_u64;
let mut billed_visit_count = 0_usize;
for (stage_id, stage) in projection.iter_stages() {
if is_exit_stage(projection, stage_id.node_id()) {
continue;
}
if stage.completion.is_none() && stage.duration_ms.is_none() && stage.usage.is_none() {
continue;
}
let node_id = stage_id.node_id();
let index = *stage_indices.entry(node_id.to_string()).or_insert_with(|| {
let index = stages.len();
stages.push(ProjectionBillingStage {
node_id: node_id.to_string(),
billing: BilledTokenCounts::default(),
duration_ms: 0,
model_id: None,
});
index
});
let row = &mut stages[index];
if let Some(duration_ms) = stage.duration_ms {
row.duration_ms = row.duration_ms.saturating_add(duration_ms);
runtime_ms = runtime_ms.saturating_add(duration_ms);
}
if let Some(usage) = stage.usage.as_ref() {
billed_visit_count += 1;
row.model_id = Some(usage.model_id().to_string());
accumulate_usage(&mut row.billing, usage);
accumulate_usage(&mut totals, usage);
let model_id = usage.model_id().to_string();
let model_entry =
by_model
.entry(model_id.clone())
.or_insert_with(|| ProjectionBillingByModel {
model_id,
stages: 0,
billing: BilledTokenCounts::default(),
});
model_entry.stages += 1;
accumulate_usage(&mut model_entry.billing, usage);
}
}
ProjectionBillingRollup {
stages,
totals,
by_model: by_model.into_values().collect(),
runtime_ms,
billed_visit_count,
}
}
fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool {
projection
.spec()
.and_then(|spec| spec.graph().nodes.get(node_id))
.is_some_and(|node| node.handler_type() == Some("exit"))
}
fn accumulate_usage(counts: &mut BilledTokenCounts, usage: &BilledModelUsage) {
let tokens = usage.tokens();
counts.input_tokens += tokens.input_tokens;
counts.output_tokens += tokens.output_tokens;
counts.reasoning_tokens += tokens.reasoning_tokens;
counts.cache_read_tokens += tokens.cache_read_tokens;
counts.cache_write_tokens += tokens.cache_write_tokens;
counts.total_tokens += tokens.total_tokens();
if let Some(value) = usage.total_usd_micros {
*counts.total_usd_micros.get_or_insert(0) += value;
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use fabro_types::{
AttrValue, BilledModelUsage, Graph, Node, RunProjection, RunSpec, StageCompletion,
StageOutcome, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
use super::billing_rollup_from_projection;
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn rollup_groups_stage_rows_by_node_and_sums_retry_visit_usage() {
let mut projection = RunProjection::default();
let failed_usage = test_usage("gpt-old", 100, 10);
let success_usage = test_usage("gpt-new", 200, 20);
let first = projection.stage_entry("verify", 1, first_event_seq(1));
first.duration_ms = Some(1200);
first.usage = Some(failed_usage);
first.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: true,
},
notes: None,
failure_reason: Some("try again".to_string()),
timestamp: chrono::Utc::now(),
});
let second = projection.stage_entry("verify", 2, first_event_seq(2));
second.duration_ms = Some(800);
second.usage = Some(success_usage);
second.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "verify");
assert_eq!(rollup.stages[0].model_id.as_deref(), Some("gpt-new"));
assert_eq!(rollup.stages[0].duration_ms, 2000);
assert_eq!(rollup.stages[0].billing.input_tokens, 300);
assert_eq!(rollup.stages[0].billing.output_tokens, 30);
assert_eq!(rollup.stages[0].billing.total_usd_micros, Some(330));
assert_eq!(rollup.runtime_ms, 2000);
assert_eq!(rollup.totals.input_tokens, 300);
assert_eq!(rollup.totals.output_tokens, 30);
assert_eq!(rollup.totals.total_usd_micros, Some(330));
assert_eq!(rollup.billed_visit_count, 2);
assert_eq!(rollup.by_model.len(), 2);
assert_eq!(rollup.by_model[0].model_id, "gpt-new");
assert_eq!(rollup.by_model[0].stages, 1);
assert_eq!(rollup.by_model[0].billing.input_tokens, 200);
assert_eq!(rollup.by_model[1].model_id, "gpt-old");
assert_eq!(rollup.by_model[1].stages, 1);
assert_eq!(rollup.by_model[1].billing.input_tokens, 100);
}
#[test]
fn rollup_includes_completed_non_llm_stage_rows_with_zero_billing() {
let mut projection = RunProjection::default();
let stage = projection.stage_entry("start", 1, first_event_seq(1));
stage.duration_ms = Some(25);
stage.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "start");
assert_eq!(rollup.stages[0].duration_ms, 25);
assert!(rollup.stages[0].model_id.is_none());
assert_eq!(rollup.stages[0].billing.input_tokens, 0);
assert_eq!(rollup.runtime_ms, 25);
assert!(rollup.by_model.is_empty());
assert!(rollup.billing_if_present().is_none());
}
#[test]
fn rollup_excludes_terminal_exit_stage_rows() {
let mut projection = RunProjection::default();
projection.spec = Some(run_spec_with_exit_node());
let start = projection.stage_entry("start", 1, first_event_seq(1));
start.duration_ms = Some(25);
start.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let exit = projection.stage_entry("exit", 1, first_event_seq(2));
exit.duration_ms = Some(7);
exit.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "start");
assert_eq!(rollup.runtime_ms, 25);
}
fn run_spec_with_exit_node() -> RunSpec {
let mut graph = Graph::new("test");
graph.nodes.insert("start".to_string(), {
let mut node = Node::new("start");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
node
});
graph.nodes.insert("exit".to_string(), {
let mut node = Node::new("exit");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
node
});
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph,
workflow_slug: None,
source_directory: None,
labels: HashMap::new(),
provenance: None,
manifest_blob: None,
definition_blob: None,
git: None,
fork_source_ref: None,
in_place: false,
}
}
}

1
lib/crates/fabro-workflow/src/error.rs
View file

@ -1838,6 +1838,7 @@ mod tests {
failure: failure.clone(),
will_retry: false,
duration_ms: 0,
billing: None,
actor: None,
};

29
lib/crates/fabro-workflow/src/event/convert.rs
View file

@ -290,12 +290,14 @@ fn event_body_from_event(event: &Event) -> EventBody {
failure,
will_retry,
duration_ms,
billing,
..
} => EventBody::StageFailed(fabro_types::StageFailedProps {
index: *index,
failure: Some(failure.clone()),
will_retry: *will_retry,
duration_ms: *duration_ms,
billing: billing.clone(),
}),
Event::StageRetrying {
index,
@ -1178,7 +1180,7 @@ mod tests {
use crate::error::Error;
use crate::event::test_support::user_principal;
use crate::event::{Event, StageScope};
use crate::outcome::FailureDetail;
use crate::outcome::{BilledModelUsage, FailureDetail};
#[derive(Debug)]
struct EventTestCause;
@ -1200,6 +1202,28 @@ mod tests {
}
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(serde_json::json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn run_event_stage_completed_places_node_fields_in_header() {
let stored = to_run_event_at(
@ -1279,6 +1303,7 @@ mod tests {
#[test]
fn run_event_stage_failure_keeps_failure_detail() {
let usage = test_usage("gpt-5.2", 321, 54);
let stored = to_run_event(&fixtures::RUN_3, &Event::StageFailed {
node_id: "code".to_string(),
name: "Code".to_string(),
@ -1289,6 +1314,7 @@ mod tests {
),
will_retry: true,
duration_ms: 5000,
billing: Some(usage.clone()),
actor: None,
});
@ -1297,6 +1323,7 @@ mod tests {
assert_eq!(properties["failure"]["message"], "lint failed");
assert_eq!(properties["failure"]["failure_class"], "deterministic");
assert_eq!(properties["will_retry"], true);
assert_eq!(properties["billing"], serde_json::to_value(&usage).unwrap());
}
#[test]

1
lib/crates/fabro-workflow/src/event/events.rs
View file

@ -206,6 +206,7 @@ pub enum Event {
failure: FailureDetail,
will_retry: bool,
duration_ms: u64,
billing: Option<BilledModelUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
actor: Option<Principal>,
},

189
lib/crates/fabro-workflow/src/lib.rs
View file

@ -20,6 +20,7 @@ use std::sync::Arc;
use fabro_retro::retro::CompletedStage;
use fabro_store::EventEnvelope;
use fabro_types::{EventBody, StageId};
/// Callback invoked when a workflow node starts executing.
pub type OnNodeCallback = Option<Arc<dyn Fn(&str) + Send + Sync>>;
@ -86,34 +87,190 @@ pub fn build_completed_stages(cp: &records::Checkpoint, run_failed: bool) -> Vec
stages
}
pub fn extract_stage_durations_from_events(events: &[EventEnvelope]) -> HashMap<String, u64> {
/// Extract the `duration_ms` from a `stage.completed` / `stage.failed`
/// event body, or `None` for any other variant.
fn stage_completion_duration_ms(body: &EventBody) -> Option<u64> {
match body {
EventBody::StageCompleted(props) => Some(props.duration_ms),
EventBody::StageFailed(props) => Some(props.duration_ms),
_ => None,
}
}
/// Extract per-stage (node_id, visit) durations from `stage.completed` /
/// `stage.failed` events. Keys on the full [`StageId`] so multi-visit stages
/// (e.g. a looped `verify` node) keep distinct durations.
///
/// This is the canonical primitive; [`total_stage_duration_by_node`] and
/// [`latest_stage_duration_by_node`] are explicit rollups built on top of it.
pub fn extract_stage_durations_by_stage_id(events: &[EventEnvelope]) -> HashMap<StageId, u64> {
let mut durations = HashMap::new();
for envelope in events {
let event = &envelope.event;
let event_name = event.event_name();
if event_name != "stage.completed" && event_name != "stage.failed" {
continue;
}
let Some(node_id) = event.node_id.as_deref() else {
let Some(duration_ms) = stage_completion_duration_ms(&envelope.event.body) else {
continue;
};
let Some(duration_ms) = event
.properties()
.ok()
.and_then(|properties| properties.get("duration_ms").cloned())
.and_then(|duration| duration.as_u64())
else {
let Some(stage_id) = envelope.event.stage_id.as_ref() else {
continue;
};
durations.insert(node_id.to_string(), duration_ms);
durations.insert(stage_id.clone(), duration_ms);
}
durations
}
/// Total duration spent in each node, summed across every visit. Use for
/// billing/usage where a retried node should count its full time.
pub fn total_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap<String, u64> {
let mut totals: HashMap<String, u64> = HashMap::new();
for (stage_id, duration_ms) in extract_stage_durations_by_stage_id(events) {
*totals.entry(stage_id.node_id().to_string()).or_default() += duration_ms;
}
totals
}
/// Duration of each node's most recent visit (the highest visit number). Use
/// for run summaries and retros where the table shows one row per node and
/// "the last attempt" is the right representative.
pub fn latest_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap<String, u64> {
let mut entries: Vec<(StageId, u64)> = extract_stage_durations_by_stage_id(events)
.into_iter()
.collect();
entries.sort_by_key(|(stage_id, _)| stage_id.visit());
let mut latest = HashMap::new();
for (stage_id, duration_ms) in entries {
latest.insert(stage_id.node_id().to_string(), duration_ms);
}
latest
}
#[cfg(test)]
mod duration_tests {
use chrono::{TimeZone, Utc};
use fabro_store::EventEnvelope;
use fabro_types::run_event::{StageCompletedProps, StageFailedProps};
use fabro_types::{EventBody, RunEvent, StageId, StageOutcome, fixtures};
use super::{
extract_stage_durations_by_stage_id, latest_stage_duration_by_node,
total_stage_duration_by_node,
};
fn completed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope {
let event = RunEvent {
id: format!("evt_{seq}"),
ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
run_id: fixtures::RUN_1,
node_id: Some(node.to_string()),
node_label: None,
stage_id: Some(StageId::new(node, visit)),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body: EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: vec![],
billing: None,
failure: None,
notes: None,
files_touched: vec![],
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
}),
};
EventEnvelope { seq, event }
}
fn failed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope {
let event = RunEvent {
id: format!("evt_{seq}"),
ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
run_id: fixtures::RUN_1,
node_id: Some(node.to_string()),
node_label: None,
stage_id: Some(StageId::new(node, visit)),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body: EventBody::StageFailed(StageFailedProps {
index: 0,
failure: None,
will_retry: true,
duration_ms,
billing: None,
}),
};
EventEnvelope { seq, event }
}
#[test]
fn extract_keys_durations_by_full_stage_id() {
let events = vec![
completed_event(1, "verify", 1, 100),
completed_event(2, "verify", 2, 200),
];
let durations = extract_stage_durations_by_stage_id(&events);
assert_eq!(
durations.get(&StageId::new("verify", 1)).copied(),
Some(100)
);
assert_eq!(
durations.get(&StageId::new("verify", 2)).copied(),
Some(200)
);
}
#[test]
fn total_sums_across_visits_per_node() {
let events = vec![
completed_event(1, "verify", 1, 100),
completed_event(2, "verify", 2, 200),
completed_event(3, "build", 1, 50),
];
let totals = total_stage_duration_by_node(&events);
assert_eq!(totals.get("verify").copied(), Some(300));
assert_eq!(totals.get("build").copied(), Some(50));
}
#[test]
fn latest_picks_highest_visit_regardless_of_input_order() {
// Visit 2 appears in the events vector before visit 1; the result
// must still reflect visit 2's duration (the latest visit).
let events = vec![
completed_event(1, "verify", 2, 999),
completed_event(2, "verify", 1, 100),
];
let latest = latest_stage_duration_by_node(&events);
assert_eq!(latest.get("verify").copied(), Some(999));
}
#[test]
fn stage_failed_durations_are_included() {
let events = vec![failed_event(1, "verify", 1, 75)];
let durations = extract_stage_durations_by_stage_id(&events);
assert_eq!(durations.get(&StageId::new("verify", 1)).copied(), Some(75));
}
}
#[doc(hidden)]
pub mod artifact;
pub mod artifact_snapshot;
pub mod artifact_upload;
pub mod billing_rollup;
pub mod command_log;
pub(crate) mod condition;
pub mod context;
@ -142,6 +299,10 @@ pub mod run_control;
pub(crate) mod run_dir;
pub mod run_lookup;
pub use billing_rollup::{
ProjectionBillingByModel, ProjectionBillingRollup, ProjectionBillingStage,
billing_rollup_from_projection,
};
pub use error::{Error, FailureCategory, FailureSignature, FailureSignatureExt, Result};
pub use manifest_path::ManifestPath;
pub mod run_materialization;

2
lib/crates/fabro-workflow/src/lifecycle/event.rs
View file

@ -224,6 +224,7 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
failure,
will_retry: true,
duration_ms,
billing: outcome.usage.clone(),
actor,
},
&scope,
@ -275,6 +276,7 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
failure,
will_retry: false,
duration_ms,
billing: outcome.usage.clone(),
actor,
},
&scope,

137
lib/crates/fabro-workflow/src/pipeline/finalize.rs
View file

@ -19,6 +19,7 @@ use crate::run_status::{FailureReason, RunStatus, SuccessReason};
use crate::runtime_store::RunStoreHandle;
use crate::sandbox_git::git_diff_with_timeout;
use crate::services::RunServices;
use crate::{ProjectionBillingRollup, billing_rollup_from_projection};
pub fn classify_engine_result(
engine_result: &Result<Outcome, Error>,
@ -68,22 +69,22 @@ pub(crate) async fn build_conclusion_from_store(
run_duration_ms: u64,
final_git_commit_sha: Option<String>,
) -> Conclusion {
let (state_result, events_result) = tokio::join!(run_store.state(), run_store.list_events());
let projection = state_result.ok();
let projection = run_store.state().await.ok();
let projection_order = projection
.as_ref()
.map(stage_projection_order)
.unwrap_or_default();
let projection_billing = projection
.as_ref()
.map(billing_rollup_from_projection)
.unwrap_or_default();
let checkpoint = projection
.as_ref()
.and_then(|state| state.checkpoint.as_ref());
let stage_durations = events_result
.map(|events| crate::extract_stage_durations_from_events(&events))
.unwrap_or_default();
build_conclusion_from_parts(
checkpoint,
&stage_durations,
&projection_billing,
&projection_order,
status,
failure_reason,
@ -94,7 +95,7 @@ pub(crate) async fn build_conclusion_from_store(
fn build_conclusion_from_parts(
checkpoint: Option<&Checkpoint>,
stage_durations: &HashMap<String, u64>,
projection_billing: &ProjectionBillingRollup,
projection_order: &HashMap<String, u32>,
status: StageOutcome,
failure_reason: Option<String>,
@ -105,6 +106,11 @@ fn build_conclusion_from_parts(
// while the other checkpoint maps are keyed by node_id. Dedupe to one row
// per node so the stages table matches the deduped billing total.
let (stages, total_retries) = if let Some(cp) = checkpoint {
let billing_by_node = projection_billing
.stages
.iter()
.map(|stage| (stage.node_id.as_str(), stage))
.collect::<HashMap<_, _>>();
let mut stage_rows = Vec::new();
let mut seen = std::collections::HashSet::new();
let mut retries_sum: u32 = 0;
@ -130,7 +136,6 @@ fn build_conclusion_from_parts(
}
for (original_checkpoint_order, node_id) in stage_order {
let outcome = cp.node_outcomes.get(node_id);
let retries = cp
.node_retries
.get(node_id)
@ -138,14 +143,13 @@ fn build_conclusion_from_parts(
.unwrap_or(1)
.saturating_sub(1);
retries_sum += retries;
let billing = billing_by_node.get(node_id);
let summary = StageSummary {
stage_id: node_id.to_string(),
stage_label: node_id.to_string(),
duration_ms: stage_durations.get(node_id).copied().unwrap_or(0),
billing_usd_micros: outcome
.and_then(|o| o.usage.as_ref())
.and_then(|usage| usage.total_usd_micros),
duration_ms: billing.map_or(0, |stage| stage.duration_ms),
billing_usd_micros: billing.and_then(|stage| stage.billing.total_usd_micros),
retries,
};
stage_rows.push((
@ -176,7 +180,7 @@ fn build_conclusion_from_parts(
failure_reason,
final_git_commit_sha,
stages,
billing: checkpoint.and_then(billing_from_checkpoint),
billing: projection_billing.billing_if_present(),
total_retries,
}
}
@ -391,15 +395,8 @@ async fn compute_final_patch(
}
}
/// Iterates `node_outcomes.values()` rather than `completed_nodes` to avoid
/// over-counting the last visit's usage on looping workflows.
pub(crate) fn billing_from_checkpoint(cp: &Checkpoint) -> Option<BilledTokenCounts> {
let usage: Vec<_> = cp
.node_outcomes
.values()
.filter_map(|o| o.usage.clone())
.collect();
(!usage.is_empty()).then(|| BilledTokenCounts::from_billed_usage(&usage))
pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option<BilledTokenCounts> {
billing_rollup_from_projection(projection).billing_if_present()
}
pub(crate) fn build_terminal_event(
@ -503,7 +500,6 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result<Con
let (final_status, failure_reason, _run_status) = classify_engine_result(&outcome);
let events = services.run_store.list_events().await.unwrap_or_default();
let stage_durations = crate::extract_stage_durations_from_events(&events);
let artifact_count = events
.iter()
.filter(|envelope| matches!(envelope.event.body, EventBody::ArtifactCaptured(_)))
@ -513,12 +509,16 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result<Con
.as_ref()
.map(stage_projection_order)
.unwrap_or_default();
let projection_billing = projection
.as_ref()
.map(billing_rollup_from_projection)
.unwrap_or_default();
let checkpoint = projection
.as_ref()
.and_then(|state| state.checkpoint.as_ref());
let conclusion = build_conclusion_from_parts(
checkpoint,
&stage_durations,
&projection_billing,
&projection_order,
final_status,
failure_reason,
@ -601,7 +601,8 @@ mod tests {
use fabro_store::{Database, EventEnvelope, RunDatabase, RunProjection};
use fabro_types::run_event::{MetadataSnapshotFailureKind, MetadataSnapshotPhase};
use fabro_types::{
EventBody, RunBlobId, RunEvent, RunId, WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, EventBody, RunBlobId, RunEvent, RunId, StageCompletion, WorkflowSettings,
first_event_seq, fixtures,
};
use object_store::memory::InMemory;
@ -737,6 +738,28 @@ mod tests {
}
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(serde_json::json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn conclusion_stage_order_follows_projection_first_event_order() {
let mut projection = RunProjection::default();
@ -753,7 +776,7 @@ mod tests {
let conclusion = build_conclusion_from_parts(
Some(&checkpoint),
&HashMap::new(),
&ProjectionBillingRollup::default(),
&projection_order,
StageOutcome::Succeeded,
None,
@ -788,7 +811,7 @@ mod tests {
let conclusion = build_conclusion_from_parts(
Some(&checkpoint),
&HashMap::new(),
&ProjectionBillingRollup::default(),
&projection_order,
StageOutcome::Succeeded,
None,
@ -804,6 +827,66 @@ mod tests {
assert_eq!(stage_ids, vec!["skipped", "finished"]);
}
#[test]
fn conclusion_billing_sums_retry_visit_usage_from_projection() {
let mut projection = RunProjection::default();
let failed_usage = test_usage("gpt-old", 100, 10);
let success_usage = test_usage("gpt-new", 200, 20);
let failed = projection.stage_entry("verify", 1, first_event_seq(1));
failed.duration_ms = Some(1200);
failed.usage = Some(failed_usage);
failed.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: true,
},
notes: None,
failure_reason: Some("try again".to_string()),
timestamp: chrono::Utc::now(),
});
let succeeded = projection.stage_entry("verify", 2, first_event_seq(2));
succeeded.duration_ms = Some(800);
succeeded.usage = Some(success_usage.clone());
succeeded.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let projection_order = stage_projection_order(&projection);
let projection_billing = billing_rollup_from_projection(&projection);
let mut latest_outcome = Outcome::success();
latest_outcome.usage = Some(success_usage);
latest_outcome.duration_ms = Some(800);
let mut checkpoint = checkpoint_with(
vec!["verify", "verify"],
HashMap::from([("verify".to_string(), latest_outcome)]),
);
checkpoint.node_retries.insert("verify".to_string(), 2);
let conclusion = build_conclusion_from_parts(
Some(&checkpoint),
&projection_billing,
&projection_order,
StageOutcome::Succeeded,
None,
10,
None,
);
assert_eq!(conclusion.billing.as_ref().unwrap().input_tokens, 300);
assert_eq!(conclusion.billing.as_ref().unwrap().output_tokens, 30);
assert_eq!(
conclusion.billing.as_ref().unwrap().total_usd_micros,
Some(330)
);
assert_eq!(conclusion.stages.len(), 1);
assert_eq!(conclusion.stages[0].stage_id, "verify");
assert_eq!(conclusion.stages[0].duration_ms, 2000);
assert_eq!(conclusion.stages[0].billing_usd_micros, Some(330));
assert_eq!(conclusion.stages[0].retries, 1);
}
fn test_services(
run_store: RunStoreHandle,
emitter: Arc<Emitter>,

2
lib/crates/fabro-workflow/src/pipeline/mod.rs
View file

@ -12,7 +12,7 @@ mod validate;
pub use execute::execute;
pub use fabro_types::PullRequestRecord;
pub(crate) use finalize::{
billing_from_checkpoint, build_conclusion_from_store, build_terminal_event,
billing_from_projection, build_conclusion_from_store, build_terminal_event,
};
pub use finalize::{classify_engine_result, finalize, write_finalize_commit};
pub use initialize::initialize;

587
lib/crates/fabro-workflow/src/pipeline/pull_request.rs
View file

@ -1,10 +1,11 @@
use std::sync::Arc;
use std::sync::{Arc, LazyLock};
use fabro_auth::CredentialSource;
use fabro_github::{self as github_app, ssh_url_to_https};
use fabro_graphviz::parser;
use fabro_llm::client::Client;
use fabro_llm::generate::{GenerateParams, generate};
use fabro_llm::generate::{GenerateParams, generate_object};
use fabro_model::Catalog;
use fabro_retro::retro::Retro;
use fabro_store::RunProjection;
use fabro_types::PullRequestRecord;
@ -18,17 +19,165 @@ use crate::outcome::{StageOutcome, format_cost as outcome_format_cost};
use crate::records::{Conclusion, RunSpec};
use crate::runtime_store::RunStoreHandle;
/// Maximum length of a PR title (Unicode scalar values). Single source of
/// truth — referenced by the structured-output schema, the system prompt,
/// and [`enforce_title_cap`].
const PR_TITLE_MAX_CHARS: usize = 72;
/// Structured output schema for the LLM-generated PR title and body.
///
/// `title` is required but allows empty strings (the only signal that
/// triggers the deterministic title fallback in
/// [`maybe_open_pull_request`]). `body` requires `minLength: 1` because
/// there is no body fallback — an empty body is fatal.
static PR_CONTENT_SCHEMA: LazyLock<serde_json::Value> = LazyLock::new(|| {
serde_json::json!({
"type": "object",
"properties": {
"title": { "type": "string", "maxLength": PR_TITLE_MAX_CHARS },
"body": { "type": "string", "minLength": 1 }
},
"required": ["title", "body"],
"additionalProperties": false
})
});
#[derive(Debug, serde::Deserialize)]
struct GeneratedPrContent {
title: String,
body: String,
}
/// System prompt that instructs the LLM how to write a Fabro PR title and
/// body. The trailing programmatic sections (Plan `<details>`, Retro,
/// Fabro Details, footer) are appended after the LLM body — the prompt
/// explicitly forbids the LLM from duplicating them.
//
// The "max 72 characters" instruction must stay in sync with
// `PR_TITLE_MAX_CHARS` and the schema above; the prompt is advisory and
// `enforce_title_cap` is the actual enforcement.
const PR_BODY_SYSTEM_PROMPT: &str = "You are writing a pull request title and description for a code change produced by an AI workflow.
OUTPUT FORMAT
Return a JSON object with exactly two fields:
- \"title\": a one-line title, max 72 characters, no trailing period.
- \"body\": the markdown body as described below.
DO NOT INCLUDE in the body
- A `#` or `##` title heading at the top — the title goes in the `title` field.
- A \"Retro\" section, \"Fabro Details\" section, cost/duration table, or \"Generated with\" footer — those are appended programmatically after your output.
- The full plan text — the full plan is appended programmatically as a <details> block.
- Bare `#1`, `#2` list prefixes — GitHub auto-links those as issue references. Use plain `1.`, `2.` instead.
- A test plan unless the testing approach is non-obvious.
SIZE THE BODY TO THE CHANGE
First classify along two axes from the diff:
- Size: how many files changed, how large the diff is.
- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns.
Then write at the matching depth:
| Profile | Body shape |
|---|---|
| Small + simple (typo, config, dep bump) | 1–2 sentences, no headers, total under ~300 characters |
| Small + non-trivial (targeted bugfix, behavioral change) | Short \"Problem / Fix\" narrative, 3–5 sentences. No headers unless two distinct concerns. |
| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. |
| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. |
| Performance improvement | Include before/after measurements if available. A markdown table works well here. |
Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better — reviewers can read the diff.
WRITING PRINCIPLES
- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*.
- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development.
- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative.
- Explain the non-obvious: spend description space on what the diff doesn't show — why this approach, what was rejected, what to look at first.
- Use structure when it earns its keep: no empty sections, no template headers without content.
- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine.
PLAN SUMMARY
The full plan is attached separately as a <details> block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes.
VISUAL AIDS
Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone — based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly.
| PR changes... | Visual aid |
|---|---|
| 3+ interacting components or services | Mermaid component / interaction diagram |
| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram |
| 3+ behavioral modes or variants | Markdown comparison table |
| Before/after data or trade-offs | Markdown table |
| Data model changes with 3+ related entities | Mermaid ERD |
Mermaid: prefer `TB` direction, ≤10 nodes typical. Place inline at the point of relevance, not in a separate \"Diagrams\" section.";
/// Truncation budget for the LLM prompt's goal / plan / diff sections.
struct TruncationCaps {
goal: usize,
plan: usize,
diff: usize,
}
/// Generous tier for models with ≥200k context windows.
const TRUNCATION_LARGE: TruncationCaps = TruncationCaps {
goal: 75_000,
plan: 75_000,
diff: 250_000,
};
/// Conservative tier (matches the pre-refactor values). Used for smaller
/// or unknown models.
const TRUNCATION_SMALL: TruncationCaps = TruncationCaps {
goal: 20_000,
plan: 20_000,
diff: 50_000,
};
/// Resolve truncation caps based on the model's context window. Unknown
/// models fall through to the conservative tier.
fn truncation_caps(model: &str) -> &'static TruncationCaps {
let large_enough = Catalog::builtin()
.get(model)
.is_some_and(|m| m.context_window() >= 200_000);
if large_enough {
&TRUNCATION_LARGE
} else {
&TRUNCATION_SMALL
}
}
/// Truncate `s` to at most `max` Unicode scalar values without splitting a
/// UTF-8 sequence.
fn truncate_chars(s: &str, max: usize) -> &str {
s.char_indices()
.nth(max)
.map_or(s, |(boundary, _)| &s[..boundary])
}
/// Truncate `s` to at most `max` Unicode scalar values, replacing the
/// trailing char with `…` when truncation occurs.
fn truncate_with_ellipsis(s: &str, max: usize) -> String {
if s.chars().count() > max {
let truncated: String = s.chars().take(max - 1).collect();
format!("{truncated}\u{2026}")
} else {
s.to_string()
}
}
/// Cap a PR title at [`PR_TITLE_MAX_CHARS`].
fn enforce_title_cap(title: &str) -> String {
truncate_with_ellipsis(title, PR_TITLE_MAX_CHARS)
}
/// Derive a PR title from the workflow goal.
///
/// Uses the first line, truncated to 120 characters for readability.
/// Uses the first line, truncated to 120 characters for readability. The
/// caller is expected to apply [`enforce_title_cap`] afterwards if a
/// stricter cap is required (the wider cap here is the legacy behaviour
/// for the deterministic fallback path).
fn pr_title_from_goal(goal: &str) -> String {
let stripped = strip_goal_decoration(goal);
if stripped.chars().count() > 120 {
let truncated: String = stripped.chars().take(119).collect();
format!("{truncated}…")
} else {
stripped.to_string()
}
truncate_with_ellipsis(strip_goal_decoration(goal), 120)
}
/// Truncate a PR body to fit GitHub's 65,536 character limit.
@ -286,8 +435,13 @@ async fn load_pull_request_diff(run_store: &RunStoreHandle) -> String {
.unwrap_or_default()
}
/// Build a complete PR body by combining LLM-generated narrative with
/// programmatic sections (plan, retro, fabro details).
/// Build a complete PR title and body by combining LLM-generated narrative
/// with programmatic sections (plan, retro, fabro details).
///
/// Returns `(title, body)`. The title may be the empty string when the LLM
/// returned a usable body but no usable title — callers fall back to
/// [`pr_title_from_goal`] in that case. Every other generation failure is
/// surfaced as `Err`.
pub async fn build_pr_body(
diff: &str,
goal: &str,
@ -295,7 +449,7 @@ pub async fn build_pr_body(
run_store: &RunStoreHandle,
llm_source: &dyn CredentialSource,
conclusion: Option<&Conclusion>,
) -> Result<String, String> {
) -> Result<(String, String), String> {
let client = Client::from_source(llm_source)
.await
.map_err(|e| format!("Failed to create LLM client: {e}"))?;
@ -310,7 +464,7 @@ async fn build_pr_body_with_client(
run_store: &RunStoreHandle,
conclusion: Option<&Conclusion>,
client: Arc<Client>,
) -> Result<String, String> {
) -> Result<(String, String), String> {
build_pr_body_with_client_and_state(diff, goal, model, run_store, conclusion, client, None)
.await
}
@ -323,7 +477,7 @@ async fn build_pr_body_with_source_and_state(
llm_source: &dyn CredentialSource,
conclusion: Option<&Conclusion>,
run_state: Option<&fabro_store::RunProjection>,
) -> Result<String, String> {
) -> Result<(String, String), String> {
let client = Client::from_source(llm_source)
.await
.map_err(|e| format!("Failed to create LLM client: {e}"))?;
@ -348,7 +502,7 @@ async fn build_pr_body_with_client_and_state(
conclusion: Option<&Conclusion>,
client: Arc<Client>,
run_state: Option<&fabro_store::RunProjection>,
) -> Result<String, String> {
) -> Result<(String, String), String> {
info!("Building PR body");
let loaded_run_state = if run_state.is_none() {
@ -369,45 +523,39 @@ async fn build_pr_body_with_client_and_state(
let run_spec = run_state.and_then(|state| state.spec.clone());
let dot_source = run_state.and_then(|state| state.graph_source.clone());
// Build LLM prompt
let system = if plan_text.is_some() {
"Write a PR description with: (1) 2-3 concise paragraphs explaining the change, then (2) a '### Plan Summary' section with bullet points summarizing the plan. Do not include a title. Do not include the full plan.".to_string()
} else {
"Write a concise PR description in 2-3 paragraphs explaining the change. Do not include a title.".to_string()
};
// Truncate diff to fit context windows (~50k chars)
let max_diff_len = 50_000;
let truncated_diff = if diff.len() > max_diff_len {
&diff[..diff.floor_char_boundary(max_diff_len)]
} else {
diff
};
let caps = truncation_caps(model);
let truncated_goal = truncate_chars(goal, caps.goal);
let truncated_diff = truncate_chars(diff, caps.diff);
let prompt = if let Some(ref plan) = plan_text {
// Truncate plan for LLM context (~20k chars)
let max_plan_len = 20_000;
let truncated_plan = if plan.len() > max_plan_len {
&plan[..plan.floor_char_boundary(max_plan_len)]
} else {
plan.as_str()
};
let truncated_plan = truncate_chars(plan, caps.plan);
format!(
"Goal: {goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```"
"Goal: {truncated_goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```"
)
} else {
format!("Goal: {goal}\n\nDiff:\n```\n{truncated_diff}\n```")
format!("Goal: {truncated_goal}\n\nDiff:\n```\n{truncated_diff}\n```")
};
let params = GenerateParams::new(model, client)
.system(system)
.system(PR_BODY_SYSTEM_PROMPT)
.prompt(prompt);
let result = generate(params)
let result = generate_object(params, PR_CONTENT_SCHEMA.clone())
.await
.map_err(|e| format!("LLM generation failed: {e}"))?;
let llm_output = result.response.text();
let output = result
.output
.ok_or_else(|| "LLM generation returned no structured output".to_string())?;
let generated: GeneratedPrContent = serde_json::from_value(output)
.map_err(|e| format!("Failed to deserialize PR content: {e}"))?;
if generated.body.trim().is_empty() {
return Err("LLM generated an empty PR body".to_string());
}
let title = enforce_title_cap(generated.title.trim());
let llm_body = generated.body;
let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default();
let arc_details_section = conclusion
@ -416,7 +564,7 @@ async fn build_pr_body_with_client_and_state(
.unwrap_or_default();
let body = assemble_pr_body(
&llm_output,
&llm_body,
plan_text.as_deref(),
&retro_section,
&arc_details_section,
@ -424,7 +572,7 @@ async fn build_pr_body_with_client_and_state(
info!("PR body generated");
Ok(body)
Ok((title, body))
}
/// Auto-merge configuration for a pull request.
@ -465,7 +613,7 @@ pub async fn maybe_open_pull_request(
let (owner, repo) =
github_app::parse_github_owner_repo(&https_url).map_err(|err| format!("{err:#}"))?;
let body = build_pr_body_with_source_and_state(
let (llm_title, body) = build_pr_body_with_source_and_state(
req.diff,
req.goal,
req.model,
@ -478,7 +626,12 @@ pub async fn maybe_open_pull_request(
.map_err(|err| format!("{err:#}"))?;
let body = truncate_pr_body(&body);
let title = pr_title_from_goal(req.goal);
let title = if llm_title.is_empty() {
pr_title_from_goal(req.goal)
} else {
llm_title
};
let title = enforce_title_cap(&title);
let created = github_app::create_pull_request(
&req.github,
@ -782,6 +935,16 @@ mod tests {
})
}
/// JSON string the MockProvider/openai mock returns to simulate the
/// structured-output response for `(title, body)`.
fn pr_content_json(title: &str, body: &str) -> String {
serde_json::to_string(&serde_json::json!({
"title": title,
"body": body,
}))
.unwrap()
}
fn make_test_conclusion() -> Conclusion {
Conclusion {
timestamp: Utc::now(),
@ -1126,17 +1289,21 @@ mod tests {
async fn build_pr_body_uses_in_memory_conclusion() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let body = build_pr_body_with_client(
let (title, body) = build_pr_body_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", "Narrative from mock."),
explicit_client(
"mock",
&pr_content_json("Mock title", "Narrative from mock."),
),
)
.await
.unwrap();
assert_eq!(title, "Mock title");
assert!(body.contains("Narrative from mock."));
assert!(body.contains("### Fabro Details"));
assert!(body.contains("Ran 3 stages in 2m 30s for $0.42"));
@ -1196,13 +1363,16 @@ mod tests {
.await
.unwrap();
let body = build_pr_body_with_client(
let (_, body) = build_pr_body_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", "Narrative from mock."),
explicit_client(
"mock",
&pr_content_json("Mock title", "Narrative from mock."),
),
)
.await
.unwrap();
@ -1283,13 +1453,16 @@ mod tests {
.await
.unwrap();
let body = build_pr_body_with_client(
let (_, body) = build_pr_body_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", "Narrative from mock."),
explicit_client(
"mock",
&pr_content_json("Mock title", "Narrative from mock."),
),
)
.await
.unwrap();
@ -1302,13 +1475,16 @@ mod tests {
async fn build_pr_body_uses_explicit_llm_client() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let body = build_pr_body_with_client(
let (_, body) = build_pr_body_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"gpt-5.4",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("openai", "Narrative from explicit client."),
explicit_client(
"openai",
&pr_content_json("Explicit title", "Narrative from explicit client."),
),
)
.await
.unwrap();
@ -1327,7 +1503,10 @@ mod tests {
.header("authorization", "Bearer vault-openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload("Narrative from vault source."));
.json_body(openai_responses_payload(&pr_content_json(
"Vault title",
"Narrative from vault source.",
)));
})
.await;
@ -1355,7 +1534,7 @@ mod tests {
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let run_store_handle: RunStoreHandle = run_store.into();
let body = build_pr_body(
let (title, body) = build_pr_body(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"gpt-5.4",
@ -1366,6 +1545,7 @@ mod tests {
.await
.unwrap();
assert_eq!(title, "Vault title");
assert!(body.contains("Narrative from vault source."));
response_mock.assert_async().await;
}
@ -1575,4 +1755,299 @@ mod tests {
assert!(diff.contains("from_store"));
}
// ── Structured-output PR content tests ──────────────────────────────
/// MockProvider returns an over-long title; builder must cap it at 72
/// chars and end with `…`. Exercises [`enforce_title_cap`] inside
/// [`build_pr_body_with_client_and_state`].
#[tokio::test]
async fn build_pr_body_truncates_long_title() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let long_title = "x".repeat(200);
let payload = pr_content_json(&long_title, "Body content.");
let (title, _) = build_pr_body_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", &payload),
)
.await
.unwrap();
assert_eq!(title.chars().count(), 72);
assert!(title.ends_with('\u{2026}'));
}
/// Empty bodies are fatal. Real providers may reject this via the
/// schema's `minLength`; the Rust-side trim check also catches it for
/// local/mock providers.
#[tokio::test]
async fn build_pr_body_returns_err_when_body_empty() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let payload = pr_content_json("Mock", "");
let result = build_pr_body_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", &payload),
)
.await;
assert!(result.is_err(), "expected Err, got {result:?}");
}
/// Whitespace-only bodies pass schema validation but fail the
/// `body.trim().is_empty()` check inside the builder.
#[tokio::test]
async fn build_pr_body_returns_err_when_body_whitespace() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let payload = pr_content_json("Mock", " \n");
let result = build_pr_body_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", &payload),
)
.await;
let err = result.expect_err("expected Err for whitespace-only body");
assert!(err.contains("empty PR body"), "unexpected error: {err}");
}
// ── maybe_open_pull_request fallback tests ──────────────────────────
/// Set of mock servers and credentials for the `maybe_open_pull_request`
/// fallback path. The builder's `Client::from_source` rebuilds the LLM
/// client from the credential source, so the in-process MockProvider
/// cannot intercept — we mock the OpenAI HTTP endpoint instead.
struct FallbackHarness {
_vault_dir: tempfile::TempDir,
// Held to keep the mock listener alive for the duration of the test;
// the test interacts with it via `Client::from_source` (which goes
// out via HTTP to the mock URL stored in `llm_source`).
openai_server: MockServer,
github_server: MockServer,
openai_mock_id: usize,
github_mock_id: usize,
llm_source: Arc<dyn CredentialSource>,
creds: fabro_github::GitHubCredentials,
run_store: RunStoreHandle,
}
impl FallbackHarness {
async fn assert_mocks_called_once(&self) {
httpmock::Mock::new(self.openai_mock_id, &self.openai_server)
.assert_async()
.await;
httpmock::Mock::new(self.github_mock_id, &self.github_server)
.assert_async()
.await;
}
}
/// Stand up an OpenAI mock that returns the given structured-output
/// payload, a GitHub mock that accepts a PR creation, a vault-backed
/// credential source, and a run store seeded with a non-empty
/// `final_patch`.
async fn setup_fallback_test_harness(openai_payload_text: &str) -> FallbackHarness {
let openai_server = MockServer::start_async().await;
let openai_mock = openai_server
.mock_async(|when, then| {
when.method(POST)
.path("/v1/responses")
.header("authorization", "Bearer vault-openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload(openai_payload_text));
})
.await;
let github_server = MockServer::start_async().await;
let github_mock = github_server
.mock_async(|when, then| {
when.method(POST)
.path("/repos/owner/repo/pulls")
.header("authorization", "Bearer test-token");
then.status(201)
.header("content-type", "application/json")
.json_body(serde_json::json!({
"number": 1,
"html_url": "https://example.test/owner/repo/pull/1",
"node_id": "PR_kwTest1",
}));
})
.await;
let vault_dir = tempfile::tempdir().unwrap();
let mut vault = Vault::load(vault_dir.path().join("secrets.json")).unwrap();
vault
.set(
"openai_codex",
&serde_json::to_string(&openai_api_key_credential("vault-openai-key")).unwrap(),
SecretType::Credential,
None,
)
.unwrap();
let base_url = openai_server.url("/v1");
let llm_source: Arc<dyn CredentialSource> =
Arc::new(VaultCredentialSource::with_env_lookup(
Arc::new(AsyncRwLock::new(vault)),
move |name| match name {
"OPENAI_BASE_URL" => Some(base_url.clone()),
_ => None,
},
));
let creds = fabro_github::GitHubCredentials::Token("test-token".to_string());
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
// Seed a non-empty `final_patch` so `load_pull_request_diff` returns
// diff content and the early-return for empty diffs does not fire.
let run_spec = RunSpec {
run_id: fixtures::RUN_1,
settings: fabro_types::WorkflowSettings::default(),
graph: Graph::new("test"),
workflow_slug: None,
source_directory: None,
git: None,
labels: HashMap::new(),
provenance: None,
manifest_blob: None,
definition_blob: None,
fork_source_ref: None,
in_place: false,
};
append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated {
run_id: fixtures::RUN_1,
settings: serde_json::to_value(&run_spec.settings).unwrap(),
graph: serde_json::to_value(&run_spec.graph).unwrap(),
workflow_source: None,
workflow_config: None,
labels: run_spec.labels.clone().into_iter().collect(),
run_dir: "/tmp/x".to_string(),
source_directory: None,
workflow_slug: None,
db_prefix: None,
provenance: None,
manifest_blob: None,
git: None,
fork_source_ref: None,
in_place: false,
web_url: None,
})
.await
.unwrap();
append_event(&run_store, &fixtures::RUN_1, &Event::WorkflowRunCompleted {
duration_ms: 1,
artifact_count: 0,
status: "succeeded".to_string(),
reason: SuccessReason::Completed,
total_usd_micros: None,
final_git_commit_sha: None,
final_patch: Some(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n".to_string(),
),
billing: None,
})
.await
.unwrap();
let openai_mock_id = openai_mock.id;
let github_mock_id = github_mock.id;
FallbackHarness {
_vault_dir: vault_dir,
openai_server,
github_server,
openai_mock_id,
github_mock_id,
llm_source,
creds,
run_store: run_store.into(),
}
}
/// LLM returns a usable body but an empty title; `maybe_open_pull_request`
/// must fall back to `pr_title_from_goal` (first line, decoration
/// stripped) and the PR creation must succeed with that title.
#[tokio::test]
async fn maybe_open_pull_request_falls_back_to_goal_title_when_llm_returns_empty_title() {
let payload = pr_content_json("", "Narrative.");
let harness = setup_fallback_test_harness(&payload).await;
let github_base_url = harness.github_server.url("");
let github = github_app::GitHubContext::new(&harness.creds, &github_base_url);
let result = maybe_open_pull_request(OpenPullRequestRequest {
github,
origin_url: "https://github.com/owner/repo.git",
base_branch: "main",
head_branch: "fabro/run/123",
goal: "Fix telemetry leak\n\ndetails...",
diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
model: "gpt-5.4",
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
conclusion: None,
run_state: None,
})
.await
.expect("PR creation should succeed");
let record = result.expect("PR record should be Some");
assert_eq!(record.title, "Fix telemetry leak");
harness.assert_mocks_called_once().await;
}
/// LLM returns an empty title; the fallback path produces a long title
/// (close to `pr_title_from_goal`'s 120-char cap), and the unconditional
/// `enforce_title_cap` in `maybe_open_pull_request` must still bring it
/// down to 72 chars ending with `…`.
#[tokio::test]
async fn maybe_open_pull_request_caps_fallback_title_at_72_chars() {
let payload = pr_content_json("", "Narrative.");
let harness = setup_fallback_test_harness(&payload).await;
let github_base_url = harness.github_server.url("");
let github = github_app::GitHubContext::new(&harness.creds, &github_base_url);
// Single ~200-char line, no `Plan:` / heading prefix, no newlines.
let goal = "x".repeat(200);
let result = maybe_open_pull_request(OpenPullRequestRequest {
github,
origin_url: "https://github.com/owner/repo.git",
base_branch: "main",
head_branch: "fabro/run/123",
goal: &goal,
diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
model: "gpt-5.4",
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
conclusion: None,
run_state: None,
})
.await
.expect("PR creation should succeed");
let record = result.expect("PR record should be Some");
assert_eq!(record.title.chars().count(), 72);
assert!(record.title.ends_with('\u{2026}'));
harness.assert_mocks_called_once().await;
}
}

2
lib/crates/fabro-workflow/src/pipeline/retro.rs
View file

@ -53,7 +53,7 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option<Retro> {
return None;
}
};
let stage_durations = crate::extract_stage_durations_from_events(&events);
let stage_durations = crate::latest_stage_duration_by_node(&events);
let mut retro = derive_retro(
options.run_id,
&options.workflow_name,

7
lib/crates/fabro-workflow/src/test_support.rs
View file

@ -17,7 +17,7 @@ use crate::handler::HandlerRegistry;
use crate::outcome::Outcome;
use crate::pipeline;
use crate::pipeline::types::{Executed, Initialized};
use crate::pipeline::{billing_from_checkpoint, build_terminal_event};
use crate::pipeline::{billing_from_projection, build_terminal_event};
use crate::records::Checkpoint;
use crate::run_metadata::RunMetadataRuntime;
use crate::run_options::RunOptions;
@ -36,10 +36,7 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed {
let executed = Box::pin(pipeline::execute(initialized.initialized)).await;
initialized.store_logger.flush().await;
let state = executed.engine.run.run_store.state().await.ok();
let billing = state
.as_ref()
.and_then(|s| s.checkpoint.as_ref())
.and_then(billing_from_checkpoint);
let billing = state.as_ref().and_then(billing_from_projection);
let event = build_terminal_event(
&executed.outcome,
executed.duration_ms,

11
lib/crates/fabro-workflow/tests/it/integration.rs
View file

@ -6809,7 +6809,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
.header("authorization", "Bearer vault-openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload("Narrative from vault source."));
.json_body(openai_responses_payload(
&serde_json::to_string(&serde_json::json!({
"title": "Vault title",
"body": "Narrative from vault source.",
}))
.unwrap(),
));
})
.await;
@ -6890,7 +6896,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
let run_store = store.open_run_reader(&run_options.run_id).await.unwrap();
let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into();
let body = fabro_workflow::pull_request::build_pr_body(
let (title, body) = fabro_workflow::pull_request::build_pr_body(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"gpt-5.4",
@ -6910,6 +6916,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
.await
.expect("PR body should build from vault-only credentials");
assert_eq!(title, "Vault title");
assert!(body.contains("Narrative from vault source."));
response_mock.assert_async().await;
}

16
lib/packages/fabro-api-client/src/api/run-internals-api.ts
View file

@ -525,10 +525,10 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config
};
},
/**
* Returns a paginated JSON list of stored run events scoped to a single workflow node (stage).
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Workflow node id (matches RunStage.id; not visit-qualified).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.
@ -962,10 +962,10 @@ export const RunInternalsApiFp = function(configuration?: Configuration) {
return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
},
/**
* Returns a paginated JSON list of stored run events scoped to a single workflow node (stage).
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Workflow node id (matches RunStage.id; not visit-qualified).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.
@ -1172,10 +1172,10 @@ export const RunInternalsApiFactory = function (configuration?: Configuration, b
return localVarFp.listStageArtifacts(id, stageId, options).then((request) => request(axios, basePath));
},
/**
* Returns a paginated JSON list of stored run events scoped to a single workflow node (stage).
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Workflow node id (matches RunStage.id; not visit-qualified).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.
@ -1372,10 +1372,10 @@ export class RunInternalsApi extends BaseAPI {
}
/**
* Returns a paginated JSON list of stored run events scoped to a single workflow node (stage).
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Workflow node id (matches RunStage.id; not visit-qualified).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.

2
lib/packages/fabro-api-client/src/models/billing-by-model.ts
View file

@ -26,7 +26,7 @@ import type { ModelReference } from './model-reference';
export interface BillingByModel {
'model': ModelReference;
/**
* Number of stages that used this model.
* Number of usage-bearing stage visits that used this model.
*/
'stages': number;
'billing': BilledTokenCounts;

2
lib/packages/fabro-api-client/src/models/billing-stage-ref.ts
View file

@ -15,7 +15,7 @@
/**
* Reference to a billing stage.
* Reference to a workflow node in a billing stage row.
*/
export interface BillingStageRef {
/**

4
lib/packages/fabro-api-client/src/models/run-billing-stage.ts
View file

@ -24,14 +24,14 @@ import type { BillingStageRef } from './billing-stage-ref';
import type { ModelReference } from './model-reference';
/**
* Token counts and billed totals for a single stage within a run.
* Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node.
*/
export interface RunBillingStage {
'stage': BillingStageRef;
'model': ModelReference | null;
'billing': BilledTokenCounts;
/**
* Wall-clock runtime in seconds.
* Wall-clock runtime in seconds, summed across every visit of this node.
*/
'runtime_secs': number;
}

2
lib/packages/fabro-api-client/src/models/run-billing.ts
View file

@ -28,7 +28,7 @@ import type { RunBillingTotals } from './run-billing-totals';
*/
export interface RunBilling {
/**
* Per-stage billing breakdown.
* Per-node billing breakdown. Each row sums billing and runtime across all visits of that node.
*/
'stages': Array<RunBillingStage>;
'totals': RunBillingTotals;

10
lib/packages/fabro-api-client/src/models/run-stage.ts
View file

@ -22,7 +22,7 @@ import type { StageState } from './stage-state';
*/
export interface RunStage {
/**
* Unique stage identifier within the run.
* StageId in \"node_id@visit\" form, e.g. verify@2.
*/
'id': string;
/**
@ -35,9 +35,13 @@ export interface RunStage {
*/
'duration_secs'?: number;
/**
* Node identifier in the Graphviz graph source.
* Node id in the workflow graph; multiple stages with different visits share the same node_id.
*/
'dot_id'?: string;
'node_id': string;
/**
* 1-based visit count; bumped each time the workflow re-enters this node.
*/
'visit': number;
}