scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-13 23:14:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge main into cancellation-token run branch

Browse source
This commit is contained in:
Bryan Helmkamp 2026-05-05 09:42:45 -04:00
commit d690a865a4
No known key found for this signature in database
105 changed files with 7978 additions and 1581 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
Cargo.lock generated
View file

@ -1536,7 +1536,7 @@ dependencies = [
[[package]]
name = "fabro-agent"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1575,7 +1575,7 @@ dependencies = [
[[package]]
name = "fabro-api"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-config",
@ -1596,7 +1596,7 @@ dependencies = [
[[package]]
name = "fabro-auth"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1620,7 +1620,7 @@ dependencies = [
[[package]]
name = "fabro-checkpoint"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-config",
@ -1636,7 +1636,7 @@ dependencies = [
[[package]]
name = "fabro-cli"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -1732,7 +1732,7 @@ dependencies = [
[[package]]
name = "fabro-client"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"bytes",
@ -1761,7 +1761,7 @@ dependencies = [
[[package]]
name = "fabro-config"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"chrono",
@ -1788,7 +1788,7 @@ dependencies = [
[[package]]
name = "fabro-core"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"fabro-types",
@ -1803,7 +1803,7 @@ dependencies = [
[[package]]
name = "fabro-dev"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -1823,7 +1823,7 @@ dependencies = [
[[package]]
name = "fabro-devcontainer"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-http",
"fabro-static",
@ -1840,7 +1840,7 @@ dependencies = [
[[package]]
name = "fabro-dump"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"bytes",
@ -1854,7 +1854,7 @@ dependencies = [
[[package]]
name = "fabro-github"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -1876,7 +1876,7 @@ dependencies = [
[[package]]
name = "fabro-graphviz"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-types",
@ -1890,7 +1890,7 @@ dependencies = [
[[package]]
name = "fabro-hooks"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"fabro-agent",
@ -1914,7 +1914,7 @@ dependencies = [
[[package]]
name = "fabro-http"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-static",
"http",
@ -1924,7 +1924,7 @@ dependencies = [
[[package]]
name = "fabro-install"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -1939,7 +1939,7 @@ dependencies = [
[[package]]
name = "fabro-interview"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"dialoguer",
@ -1954,7 +1954,7 @@ dependencies = [
[[package]]
name = "fabro-llm"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1986,7 +1986,7 @@ dependencies = [
[[package]]
name = "fabro-macros"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"clap",
"fabro-options-metadata",
@ -1997,7 +1997,7 @@ dependencies = [
[[package]]
name = "fabro-mcp"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-config",
@ -2013,7 +2013,7 @@ dependencies = [
[[package]]
name = "fabro-model"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-static",
"insta",
@ -2024,7 +2024,7 @@ dependencies = [
[[package]]
name = "fabro-oauth"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"axum",
@ -2046,7 +2046,7 @@ dependencies = [
[[package]]
name = "fabro-options-metadata"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"serde",
"serde_json",
@ -2054,7 +2054,7 @@ dependencies = [
[[package]]
name = "fabro-proc"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"cc",
"libc",
@ -2063,7 +2063,7 @@ dependencies = [
[[package]]
name = "fabro-redact"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"aho-corasick",
"ref-cast",
@ -2079,7 +2079,7 @@ dependencies = [
[[package]]
name = "fabro-retro"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"chrono",
@ -2098,7 +2098,7 @@ dependencies = [
[[package]]
name = "fabro-sandbox"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2138,7 +2138,7 @@ dependencies = [
[[package]]
name = "fabro-server"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2219,7 +2219,7 @@ dependencies = [
[[package]]
name = "fabro-slack"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-http",
"fabro-interview",
@ -2240,18 +2240,18 @@ dependencies = [
[[package]]
name = "fabro-spa"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"rust-embed",
]
[[package]]
name = "fabro-static"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
[[package]]
name = "fabro-store"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"bytes",
@ -2278,7 +2278,7 @@ dependencies = [
[[package]]
name = "fabro-telemetry"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -2304,7 +2304,7 @@ dependencies = [
[[package]]
name = "fabro-template"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-util",
@ -2316,7 +2316,7 @@ dependencies = [
[[package]]
name = "fabro-test"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"assert_cmd",
"axum",
@ -2339,7 +2339,7 @@ dependencies = [
[[package]]
name = "fabro-tracker"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2353,7 +2353,7 @@ dependencies = [
[[package]]
name = "fabro-types"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"clap",
@ -2374,7 +2374,7 @@ dependencies = [
[[package]]
name = "fabro-util"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"console 0.15.11",
@ -2394,7 +2394,7 @@ dependencies = [
[[package]]
name = "fabro-validate"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-graphviz",
"fabro-model",
@ -2404,7 +2404,7 @@ dependencies = [
[[package]]
name = "fabro-vault"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-types",
@ -2416,7 +2416,7 @@ dependencies = [
[[package]]
name = "fabro-workflow"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -7120,7 +7120,7 @@ dependencies = [
[[package]]
name = "twin-github"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"axum",
"base64",
@ -7139,7 +7139,7 @@ dependencies = [
[[package]]
name = "twin-openai"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-stream",

2
Cargo.toml
View file

@ -5,7 +5,7 @@ resolver = "2"
[workspace.package]
edition = "2021"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
license = "MIT"
[workspace.dependencies]

20
apps/fabro-web/app/components/stage-sidebar.tsx
View file

@ -1,4 +1,4 @@
import { useState, useEffect, useRef, type ComponentType } from "react";
import { useEffect, useRef, type ComponentType } from "react";
import { Link } from "react-router";
import type { StageState } from "@qltysh/fabro-api-client";
import {
@ -11,14 +11,16 @@ import {
} from "@heroicons/react/24/solid";
import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline";
import { formatDurationSecs } from "../lib/format";
import { ACTIVE_STAGE_STATES } from "../lib/stage-sidebar";
import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar";
import { useTickingNow } from "../lib/time";
export interface Stage {
id: string;
name: string;
status: StageState;
duration: string;
dotId?: string;
nodeId: string;
visit: number;
}
export const statusConfig: Record<StageState, { icon: ComponentType<{ className?: string }>; color: string }> = {
@ -42,7 +44,6 @@ interface StageSidebarProps {
export function StageSidebar({ stages, runId, selectedStageId, activeLink }: StageSidebarProps) {
// Track when we first observed each running stage (for ticking timer)
const runningStartRef = useRef<Map<string, number>>(new Map());
const [, setTick] = useState(0);
// Track start times for running stages
useEffect(() => {
@ -62,16 +63,13 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta
}, [stages]);
// Tick every second while any stage is running
useEffect(() => {
if (!stages.some((s) => ACTIVE_STAGE_STATES.has(s.status))) return;
const interval = setInterval(() => setTick((t) => t + 1), 1000);
return () => clearInterval(interval);
}, [stages]);
const hasActive = stages.some((s) => ACTIVE_STAGE_STATES.has(s.status));
const now = useTickingNow(hasActive);
function stageDuration(stage: Stage): string {
if (ACTIVE_STAGE_STATES.has(stage.status)) {
const start = runningStartRef.current.get(stage.id);
if (start) return formatDurationSecs(Math.floor((Date.now() - start) / 1000));
if (start) return formatDurationSecs(Math.floor((now - start) / 1000));
return "0s";
}
return stage.duration;
@ -100,7 +98,7 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta
}`}
>
<Icon className={`size-4 shrink-0 ${config.color} ${ACTIVE_STAGE_STATES.has(stage.status) ? "animate-spin" : ""}`} />
<span className="flex-1 truncate">{stage.name}</span>
<span className="flex-1 truncate">{formatStageLabel(stage)}</span>
<span className="font-mono text-xs tabular-nums text-fg-muted">{stageDuration(stage)}</span>
</Link>
</li>

23
apps/fabro-web/app/data/runs.ts
View file

@ -1,8 +1,10 @@
import { formatElapsedSecs, formatDurationSecs } from "../lib/format";
import type {
RunListItem,
RunStatus as ApiRunStatus,
RunSummary,
import {
BoardColumn,
type BoardColumn as ApiBoardColumn,
type RunListItem,
type RunStatus as ApiRunStatus,
type RunSummary,
} from "@qltysh/fabro-api-client";
export type CiStatus = "passing" | "failing" | "pending";
@ -37,9 +39,19 @@ export interface RunItem {
sourceDirectory?: string;
}
export type ColumnStatus = "initializing" | "running" | "blocked" | "succeeded" | "failed";
export type ColumnStatus = ApiBoardColumn;
export const columnStatuses = [
BoardColumn.QUEUED,
BoardColumn.INITIALIZING,
BoardColumn.RUNNING,
BoardColumn.BLOCKED,
BoardColumn.SUCCEEDED,
BoardColumn.FAILED,
] as const satisfies readonly ColumnStatus[];
export const columnStatusDisplay: Record<ColumnStatus, { label: string; dot: string; text: string }> = {
queued: { label: "Queued", dot: "bg-fg-muted", text: "text-fg-muted" },
initializing: { label: "Initializing", dot: "bg-amber", text: "text-amber" },
running: { label: "Running", dot: "bg-teal-500", text: "text-teal-500" },
blocked: { label: "Blocked", dot: "bg-amber", text: "text-amber" },
@ -113,6 +125,7 @@ export function columnForStatus(status: ApiRunStatus | null | undefined): Column
switch (status?.kind) {
case "submitted":
case "queued":
return "queued";
case "starting":
return "initializing";
case "running":

65
apps/fabro-web/app/lib/api-client.ts
View file

@ -217,6 +217,69 @@ export async function apiPaginatedFetcher<TItem, TExtra extends object = {}>(
}
}
function stageEventsPagePath(key: string, sinceSeq: number, limit: number): string {
const url = new URL(apiPath(key), "http://fabro.local");
url.searchParams.set("since_seq", String(sinceSeq));
url.searchParams.set("limit", String(limit));
return `${url.pathname}${url.search}`;
}
/**
* Cursor-paginated fetcher for `/runs/{id}/stages/{stageId}/events`.
*
* Loops from `since_seq=1` with a 1000-event page size, advancing the cursor
* to `highestSeq + 1` until the server reports `meta.has_more === false`.
* The empty-page guard mirrors `apiPaginatedFetcher`: if the server claims
* `has_more` but returns no rows we exit and `console.warn` to surface the
* server invariant violation without spinning the UI.
*/
export async function fetchAllStageEvents<TItem extends { seq: number }>(
key: string,
): Promise<TItem[]> {
const PAGE_LIMIT = 1000;
const MAX_PAGES = 50;
const data: TItem[] = [];
let sinceSeq = 1;
let pagesLoaded = 0;
while (true) {
const response = await apiRequest(stageEventsPagePath(key, sinceSeq, PAGE_LIMIT));
if (!response.ok) {
throw await apiErrorFromResponse(response);
}
const page = (await response.json()) as PaginatedEnvelope<TItem>;
pagesLoaded += 1;
if (page.data.length === 0) {
if (page.meta.has_more) {
console.warn(
`Stage events fetch for ${key} returned an empty page with has_more=true; stopping at ${data.length} items to avoid spinning.`,
);
}
return data;
}
data.push(...page.data);
if (!page.meta.has_more) return data;
if (pagesLoaded >= MAX_PAGES) {
console.warn(
`Stopped stage events fetch for ${key} after ${pagesLoaded} pages and ${data.length} items because the safety cap was reached.`,
);
return data;
}
const highestSeq = page.data.reduce((max, event) => Math.max(max, event.seq), sinceSeq - 1);
if (highestSeq < sinceSeq) {
console.warn(
`Stage events fetch for ${key} returned a non-advancing page at since_seq=${sinceSeq}; stopping at ${data.length} items to avoid spinning.`,
);
return data;
}
sinceSeq = highestSeq + 1;
}
}
export async function apiJsonMutation<TResponse, TArg = unknown>(
key: string,
{ arg }: { arg: TArg },
@ -233,4 +296,4 @@ export async function apiJsonMutation<TResponse, TArg = unknown>(
}
if (response.status === 204) return undefined as TResponse;
return response.json() as Promise<TResponse>;
}
}

94
apps/fabro-web/app/lib/board-events.test.tsx
View file

@ -4,7 +4,12 @@ import {
shouldRefreshBoardForEvent,
subscribeToBoardEvents,
} from "./board-events";
import {
createCrossTabSseCoordinator,
type BroadcastChannelLike,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import type { EventSourceLike } from "./sse";
type MessageHandler = ((event: { data: string }) => void) | null;
@ -21,6 +26,14 @@ class FakeEventSource {
}
}
class FakeBroadcastChannel implements BroadcastChannelLike {
onmessage: ((event: { data: unknown }) => void) | null = null;
postMessage() {}
close() {}
}
describe("shouldRefreshBoardForEvent", () => {
test("refreshes board for run and interview status changes only", () => {
expect(shouldRefreshBoardForEvent("run.running")).toBe(true);
@ -31,22 +44,28 @@ describe("shouldRefreshBoardForEvent", () => {
});
describe("subscribeToBoardEvents", () => {
test("shares one source and invalidates the board runs key", () => {
test("coordinated mode shares one global source and invalidates the board runs key", async () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createCoordinator((url) => {
created.push(url);
return source;
});
const mutate = (key: string) => {
keys.push(key);
return Promise.resolve();
};
const firstCleanup = subscribeToBoardEvents(mutate, (url) => {
created.push(url);
return source;
}, { debounceMs: 0 });
const firstCleanup = subscribeToBoardEvents(mutate, () => {
throw new Error("source should be created by coordinator");
}, { debounceMs: 0, coordinator });
const secondCleanup = subscribeToBoardEvents(mutate, () => {
throw new Error("source should be reused");
}, { debounceMs: 0 });
}, { debounceMs: 0, coordinator });
await waitFor(() => created.length === 1);
keys.length = 0;
source.emit({ event: "run.running" });
@ -57,5 +76,68 @@ describe("subscribeToBoardEvents", () => {
expect(source.closed).toBe(false);
secondCleanup();
expect(source.closed).toBe(true);
coordinator.close();
});
test("fallback mode preserves the existing shared board EventSource", () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const mutate = (key: string) => {
keys.push(key);
return Promise.resolve();
};
const firstCleanup = subscribeToBoardEvents(mutate, (url) => {
created.push(url);
return source;
}, { debounceMs: 0, coordinator });
const secondCleanup = subscribeToBoardEvents(mutate, () => {
throw new Error("source should be reused");
}, { debounceMs: 0, coordinator });
source.emit({ event: "run.running" });
expect(created).toEqual(["/api/v1/attach"]);
expect(keys).toEqual([queryKeys.boards.runs()]);
firstCleanup();
expect(source.closed).toBe(false);
secondCleanup();
expect(source.closed).toBe(true);
coordinator.close();
});
});
function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) {
return createCrossTabSseCoordinator({
tabId: "board-test",
channelFactory: () => new FakeBroadcastChannel(),
eventSourceFactory,
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: {
heartbeatMs: 10,
leaderStaleMs: 50,
electionJitterMs: 0,
},
});
}
function createFallbackCoordinator() {
return createCrossTabSseCoordinator({
channelFactory: () => {
throw new Error("BroadcastChannel unavailable");
},
});
}
async function waitFor(condition: () => boolean, timeoutMs = 200) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (condition()) return;
await new Promise((resolve) => setTimeout(resolve, 2));
}
throw new Error("condition did not become true before timeout");
}

42
apps/fabro-web/app/lib/board-events.ts
View file

@ -1,6 +1,10 @@
import { useEffect } from "react";
import { useSWRConfig } from "swr";
import {
subscribeToCrossTabSse,
type CrossTabSseCoordinator,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import {
createBrowserEventSource,
@ -11,6 +15,11 @@ import {
type SharedEventSubscription,
} from "./sse";
interface BoardEventOptions {
debounceMs?: number;
coordinator?: CrossTabSseCoordinator;
}
const BOARD_STATUS_EVENTS = new Set([
"run.submitted",
"run.queued",
@ -41,23 +50,36 @@ export function shouldRefreshBoardForEvent(event: string) {
export function subscribeToBoardEvents(
mutate: MutateFn,
eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource,
{ debounceMs = 500 }: { debounceMs?: number } = {},
{ debounceMs = 500, coordinator }: BoardEventOptions = {},
): () => void {
return subscribeToSharedEventSource<EventPayload>({
subscriptions,
return subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey: BOARD_SUBSCRIPTION_KEY,
url: queryKeys.system.attach(),
mutate,
eventSourceFactory,
debounceMs,
resolveInvalidation: (payload) => ({
keys: payload.event && shouldRefreshBoardForEvent(payload.event)
? [queryKeys.boards.runs()]
: [],
}),
resyncKeys: () => [queryKeys.boards.runs()],
resolveInvalidation: boardInvalidation,
fallbackSubscribe: () =>
subscribeToSharedEventSource<EventPayload>({
subscriptions,
subscriptionKey: BOARD_SUBSCRIPTION_KEY,
url: queryKeys.system.attach(),
mutate,
eventSourceFactory,
debounceMs,
resolveInvalidation: boardInvalidation,
}),
});
}
function boardInvalidation(payload: EventPayload) {
return {
keys: payload.event && shouldRefreshBoardForEvent(payload.event)
? [queryKeys.boards.runs()]
: [],
};
}
export function useBoardEvents() {
const { mutate } = useSWRConfig();

736
apps/fabro-web/app/lib/cross-tab-sse.test.ts Normal file
View file

@ -0,0 +1,736 @@
import { afterEach, describe, expect, test } from "bun:test";
import {
CROSS_TAB_SSE_CHANNEL,
createCrossTabSseCoordinator,
subscribeToCrossTabSse,
type BroadcastChannelLike,
type CrossTabSseCoordinator,
type CrossTabSseMessage,
} from "./cross-tab-sse";
import type { EventPayload, MutateFn } from "./sse";
type MessageHandler = ((event: { data: string }) => void) | null;
type TabVisibility = "visible" | "hidden";
const TEST_TIMING = {
heartbeatMs: 10,
leaderStaleMs: 35,
electionJitterMs: 5,
};
class FakeEventSource {
onmessage: MessageHandler = null;
closed = false;
constructor(
readonly url: string,
readonly owner: string,
) {}
emit(payload: unknown) {
this.onmessage?.({ data: JSON.stringify(payload) });
}
close() {
this.closed = true;
}
}
class FakeBroadcastChannel implements BroadcastChannelLike {
static channels = new Set<FakeBroadcastChannel>();
static muted = false;
static throwOnTypes = new Set<CrossTabSseMessage["type"]>();
onmessage: ((event: { data: unknown }) => void) | null = null;
closed = false;
constructor(readonly name: string) {
FakeBroadcastChannel.channels.add(this);
}
postMessage(message: CrossTabSseMessage) {
if (FakeBroadcastChannel.throwOnTypes.has(message.type)) {
throw new Error(`postMessage failed for ${message.type}`);
}
if (FakeBroadcastChannel.muted) return;
const recipients = [...FakeBroadcastChannel.channels].filter(
(channel) => channel !== this && !channel.closed && channel.name === this.name,
);
queueMicrotask(() => {
for (const channel of recipients) {
if (channel.closed) continue;
channel.onmessage?.({ data: { ...message } });
}
});
}
static broadcastExternal(message: CrossTabSseMessage) {
queueMicrotask(() => {
for (const channel of FakeBroadcastChannel.channels) {
if (channel.closed) continue;
channel.onmessage?.({ data: { ...message } });
}
});
}
close() {
this.closed = true;
FakeBroadcastChannel.channels.delete(this);
}
static reset() {
for (const channel of FakeBroadcastChannel.channels) {
channel.closed = true;
}
FakeBroadcastChannel.channels.clear();
FakeBroadcastChannel.muted = false;
FakeBroadcastChannel.throwOnTypes.clear();
}
}
class Harness {
readonly sources: FakeEventSource[] = [];
readonly coordinators = new Map<string, CrossTabSseCoordinator>();
readonly visibility = new Map<string, TabVisibility>();
readonly visibilityHandlers = new Map<string, () => void>();
now = 1000;
createTab(tabId: string, visibility: TabVisibility = "visible") {
this.visibility.set(tabId, visibility);
const coordinator = createCrossTabSseCoordinator({
tabId,
channelFactory: (name) => new FakeBroadcastChannel(name),
eventSourceFactory: (url) => {
const source = new FakeEventSource(url, tabId);
this.sources.push(source);
return source;
},
getVisibility: () => this.visibility.get(tabId) ?? "visible",
addVisibilityChangeListener: (handler) => {
this.visibilityHandlers.set(tabId, handler);
return () => this.visibilityHandlers.delete(tabId);
},
addPagehideListener: () => () => {},
now: () => this.now,
timing: TEST_TIMING,
});
this.coordinators.set(tabId, coordinator);
return coordinator;
}
setVisibility(tabId: string, visibility: TabVisibility) {
this.visibility.set(tabId, visibility);
this.visibilityHandlers.get(tabId)?.();
}
openSources() {
return this.sources.filter((source) => !source.closed);
}
close() {
for (const coordinator of this.coordinators.values()) {
coordinator.close();
}
}
}
const harnesses: Harness[] = [];
afterEach(() => {
for (const harness of harnesses.splice(0)) {
harness.close();
}
FakeBroadcastChannel.reset();
});
describe("subscribeToCrossTabSse", () => {
test("opens one leader-owned global EventSource and keeps followers passive", async () => {
const harness = newHarness();
const cleanups = ["a", "b", "c"].map((tabId) => {
const coordinator = harness.createTab(tabId);
return subscribeForRunEvent(coordinator, []);
});
await waitFor(() => harness.openSources().length === 1);
expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]);
expect([...FakeBroadcastChannel.channels].every((channel) => channel.name === CROSS_TAB_SSE_CHANNEL)).toBe(true);
cleanups.forEach((cleanup) => cleanup());
});
test("leader broadcasts events to all local subscribers", async () => {
const harness = newHarness();
const keysByTab = new Map<string, string[]>();
for (const tabId of ["a", "b", "c"]) {
keysByTab.set(tabId, []);
subscribeForRunEvent(harness.createTab(tabId), keysByTab.get(tabId)!);
}
await waitFor(() => harness.openSources().length === 1);
clearRecordedKeys(keysByTab);
harness.openSources()[0].emit(runEvent({ id: "evt-1", runId: "run-1", seq: 1 }));
await waitFor(() => [...keysByTab.values()].every((keys) => keys.length === 1));
expect(keysByTab.get("a")).toEqual(["event"]);
expect(keysByTab.get("b")).toEqual(["event"]);
expect(keysByTab.get("c")).toEqual(["event"]);
});
test("board and run subscriptions coexist on the same global stream", async () => {
const harness = newHarness();
const coordinator = harness.createTab("a");
const boardKeys: string[] = [];
const runKeys: string[] = [];
subscribeForEvent(coordinator, {
subscriptionKey: "board",
keys: boardKeys,
resolveInvalidation: (payload) => ({
keys: payload.event === "run.running" ? ["board"] : [],
}),
resyncKeys: () => ["board-resync"],
});
subscribeForEvent(coordinator, {
subscriptionKey: "run:run-1",
keys: runKeys,
resolveInvalidation: (payload) => ({
keys: payload.event === "run.running" && payload.run_id === "run-1" ? ["run"] : [],
}),
resyncKeys: () => ["run-resync"],
});
await waitFor(() => harness.openSources().length === 1);
boardKeys.length = 0;
runKeys.length = 0;
harness.openSources()[0].emit(runEvent({ id: "evt-coexist", runId: "run-1", seq: 1 }));
expect(boardKeys).toEqual(["board"]);
expect(runKeys).toEqual(["run"]);
expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]);
});
test("dedupes duplicate event ids until TTL or max-size eviction", async () => {
const harness = newHarness();
const keys: string[] = [];
subscribeForRunEvent(harness.createTab("a"), keys);
await waitFor(() => harness.openSources().length === 1);
keys.length = 0;
const source = harness.openSources()[0];
source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 }));
source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 }));
expect(keys).toEqual(["event"]);
harness.now += 5 * 60 * 1000 + 1;
source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 }));
expect(keys).toEqual(["event", "event"]);
keys.length = 0;
for (let i = 0; i < 1001; i += 1) {
source.emit(runEvent({ id: `evt-${i}`, runId: "run-1", seq: i + 2 }));
}
source.emit(runEvent({ id: "evt-0", runId: "run-1", seq: 2 }));
expect(keys).toHaveLength(1002);
});
test("visible followers take over from a fresh hidden leader and resync", async () => {
const harness = newHarness();
const hiddenKeys: string[] = [];
const visibleKeys: string[] = [];
subscribeForRunEvent(harness.createTab("z", "hidden"), hiddenKeys);
await waitFor(() => harness.openSources().length === 1);
const hiddenSource = harness.openSources()[0];
subscribeForRunEvent(harness.createTab("a", "visible"), visibleKeys);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
expect(hiddenSource.closed).toBe(true);
expect(visibleKeys).toContain("resync");
});
test("visible candidates racing for the same hidden leader resolve lexically", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("b", "visible"), []);
subscribeForRunEvent(harness.createTab("a", "visible"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== "z");
expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]);
});
test("a lower lexical follower does not preempt a fresh visible leader", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "visible"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("a", "visible"), []);
await sleep(TEST_TIMING.electionJitterMs * 4);
expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]);
});
test("stale leader detection opens a new leader source and resyncs followers", async () => {
const harness = newHarness();
const followerKeys: string[] = [];
subscribeForRunEvent(harness.createTab("a"), []);
subscribeForRunEvent(harness.createTab("b"), followerKeys);
await waitFor(() => harness.openSources().length === 1);
const staleLeader = harness.openSources()[0];
harness.coordinators.get(staleLeader.owner)?.close();
harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1;
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== staleLeader.owner);
expect(followerKeys).toContain("resync");
});
test("simultaneous stale leader elections resolve to the lexical winner", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("b"), []);
subscribeForRunEvent(harness.createTab("a"), []);
await sleep(TEST_TIMING.heartbeatMs * 2);
harness.coordinators.get("z")?.close();
harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1;
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
});
test("hidden leader ignores candidates for old observed leadership", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
const hiddenSource = harness.openSources()[0];
FakeBroadcastChannel.broadcastExternal({
type: "candidate",
version: 1,
tabId: "ghost",
sentAt: harness.now,
candidateId: "ghost",
candidateGeneration: 1,
visibility: "visible",
observedLeaderId: "z",
observedGeneration: 0,
reason: "hidden-leader",
});
await sleep(TEST_TIMING.electionJitterMs * 2);
expect(hiddenSource.closed).toBe(false);
expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]);
});
test("prunes candidate records from older generations", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
const coordinator = harness.createTab("a", "visible");
subscribeForRunEvent(coordinator, []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
FakeBroadcastChannel.broadcastExternal({
type: "candidate",
version: 1,
tabId: "old-candidate",
sentAt: harness.now,
candidateId: "old-candidate",
candidateGeneration: 1,
visibility: "visible",
observedLeaderId: "previous-leader",
observedGeneration: 0,
reason: "stale-leader",
});
await sleep(TEST_TIMING.electionJitterMs * 2);
expect(candidateGenerations(coordinator)).not.toContain(1);
});
test("same-generation split brain converges to the higher-priority visible leader", async () => {
const harness = newHarness();
FakeBroadcastChannel.muted = true;
subscribeForRunEvent(harness.createTab("b"), []);
subscribeForRunEvent(harness.createTab("a"), []);
await waitFor(() => harness.openSources().length === 2);
FakeBroadcastChannel.muted = false;
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
});
test("old leader events are ignored after takeover", async () => {
const harness = newHarness();
const keys: string[] = [];
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1);
const oldSource = harness.openSources()[0];
subscribeForRunEvent(harness.createTab("a", "visible"), keys);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
keys.length = 0;
oldSource.emit(runEvent({ id: "evt-old", runId: "run-1", seq: 1 }));
expect(keys).toEqual([]);
});
test("old leader heartbeats are ignored after takeover", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("a", "visible"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
FakeBroadcastChannel.broadcastExternal({
type: "heartbeat",
version: 1,
tabId: "z",
sentAt: harness.now,
leaderId: "z",
generation: 1,
visibility: "hidden",
});
await sleep(TEST_TIMING.heartbeatMs * 2);
expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]);
});
test("last unsubscribe closes the leader source and releases leadership", async () => {
const harness = newHarness();
const cleanup = subscribeForRunEvent(harness.createTab("a"), []);
await waitFor(() => harness.openSources().length === 1);
const source = harness.openSources()[0];
cleanup();
expect(source.closed).toBe(true);
expect(harness.openSources()).toEqual([]);
});
test("missing BroadcastChannel uses subscriber fallback", () => {
const coordinator = createCrossTabSseCoordinator({
channelFactory: () => {
throw new Error("no channel");
},
});
let fallbackStarted = 0;
let fallbackStopped = 0;
const cleanup = subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey: "fallback",
mutate: (() => Promise.resolve()) as MutateFn,
resolveInvalidation: () => ({ keys: [] }),
resyncKeys: () => [],
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
debounceMs: 0,
});
cleanup();
expect(fallbackStarted).toBe(1);
expect(fallbackStopped).toBe(1);
});
test("postMessage failure after initialization degrades to fallback without coordinated resync", async () => {
const harness = newHarness();
const coordinator = harness.createTab("a");
const keys: string[] = [];
let fallbackStarted = 0;
let fallbackStopped = 0;
FakeBroadcastChannel.throwOnTypes.add("leader-changed");
const cleanup = subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey: "throwing-channel",
mutate: ((key: string) => {
keys.push(key);
return Promise.resolve();
}) as MutateFn,
resolveInvalidation: () => ({ keys: ["event"] }),
resyncKeys: () => ["resync"],
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
debounceMs: 0,
});
await waitFor(() => fallbackStarted === 1);
expect(harness.openSources()).toEqual([]);
expect(keys).toEqual([]);
cleanup();
expect(fallbackStopped).toBe(1);
});
test("close resets coordination availability after an initial channel failure", async () => {
let channelUnavailable = true;
const sources: FakeEventSource[] = [];
const coordinator = createCrossTabSseCoordinator({
tabId: "recovering",
channelFactory: (name) => {
if (channelUnavailable) throw new Error("channel unavailable");
return new FakeBroadcastChannel(name);
},
eventSourceFactory: (url) => {
const source = new FakeEventSource(url, "recovering");
sources.push(source);
return source;
},
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: TEST_TIMING,
});
let firstFallbackStarted = 0;
let secondFallbackStarted = 0;
const firstCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
firstFallbackStarted += 1;
return () => {};
},
});
firstCleanup();
coordinator.close();
channelUnavailable = false;
const secondCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
secondFallbackStarted += 1;
return () => {};
},
});
await waitFor(() => sources.some((source) => !source.closed));
expect(firstFallbackStarted).toBe(1);
expect(secondFallbackStarted).toBe(0);
expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]);
secondCleanup();
coordinator.close();
});
test("last unsubscribe retries coordination after an initial channel failure", async () => {
let channelUnavailable = true;
const sources: FakeEventSource[] = [];
const coordinator = createCrossTabSseCoordinator({
tabId: "retry-after-unsubscribe",
channelFactory: (name) => {
if (channelUnavailable) throw new Error("channel unavailable");
return new FakeBroadcastChannel(name);
},
eventSourceFactory: (url) => {
const source = new FakeEventSource(url, "retry-after-unsubscribe");
sources.push(source);
return source;
},
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: TEST_TIMING,
});
let fallbackStarted = 0;
const firstCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {};
},
});
firstCleanup();
channelUnavailable = false;
const secondCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {};
},
});
await waitFor(() => sources.some((source) => !source.closed));
expect(fallbackStarted).toBe(1);
expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]);
secondCleanup();
coordinator.close();
});
test("close stops fallback subscriptions added after degradation", async () => {
const harness = newHarness();
const coordinator = harness.createTab("a");
let fallbackStarted = 0;
let fallbackStopped = 0;
FakeBroadcastChannel.throwOnTypes.add("leader-changed");
subscribeWithFallback(coordinator, {
subscriptionKey: "before-degrade",
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
});
await waitFor(() => fallbackStarted === 1);
FakeBroadcastChannel.throwOnTypes.clear();
subscribeWithFallback(coordinator, {
subscriptionKey: "after-degrade",
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
});
expect(fallbackStarted).toBe(2);
coordinator.close();
expect(fallbackStopped).toBe(2);
});
});
function newHarness() {
const harness = new Harness();
harnesses.push(harness);
return harness;
}
function subscribeForRunEvent(coordinator: CrossTabSseCoordinator, keys: string[]) {
return subscribeForEvent(coordinator, {
subscriptionKey: "run-feed",
keys,
resolveInvalidation: (payload) => ({
keys: payload.event === "run.running" ? ["event"] : [],
}),
resyncKeys: () => ["resync"],
});
}
function subscribeForEvent(
coordinator: CrossTabSseCoordinator,
{
subscriptionKey,
keys,
resolveInvalidation,
resyncKeys,
}: {
subscriptionKey: string;
keys: string[];
resolveInvalidation: (payload: EventPayload) => { keys: string[] };
resyncKeys: () => string[];
},
) {
return subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey,
mutate: ((key: string) => {
keys.push(key);
return Promise.resolve();
}) as MutateFn,
resolveInvalidation,
resyncKeys,
fallbackSubscribe: () => {
throw new Error("fallback should not be used");
},
debounceMs: 0,
});
}
function subscribeWithFallback(
coordinator: CrossTabSseCoordinator,
{
subscriptionKey = "fallback-test",
fallbackSubscribe,
}: {
subscriptionKey?: string;
fallbackSubscribe: () => () => void;
},
) {
return subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey,
mutate: (() => Promise.resolve()) as MutateFn,
resolveInvalidation: () => ({ keys: [] }),
resyncKeys: () => [],
fallbackSubscribe,
debounceMs: 0,
});
}
function candidateGenerations(coordinator: CrossTabSseCoordinator): number[] {
const inspectable = coordinator as unknown as {
candidates: Map<string, { candidateGeneration: number }>;
};
return [...inspectable.candidates.values()].map((candidate) => candidate.candidateGeneration);
}
function runEvent({
id,
runId,
seq,
}: {
id: string;
runId: string;
seq: number;
}) {
return {
id,
seq,
run_id: runId,
event: "run.running",
ts: "2026-05-04T12:00:00.000Z",
};
}
async function waitFor(condition: () => boolean, timeoutMs = 500) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (condition()) return;
await sleep(2);
}
throw new Error("condition did not become true before timeout");
}
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function clearRecordedKeys(keysByTab: Map<string, string[]>) {
for (const keys of keysByTab.values()) {
keys.length = 0;
}
}

1175
apps/fabro-web/app/lib/cross-tab-sse.ts Normal file
View file

File diff suppressed because it is too large Load diff

34
apps/fabro-web/app/lib/queries.ts
View file

@ -1,12 +1,11 @@
import useSWR, { type SWRConfiguration } from "swr";
import type {
ApiQuestion,
EventEnvelope,
PaginatedBoardRunList,
PaginatedEventList,
PaginatedRunFileList,
PaginatedRunList,
PaginatedRunStageList,
PaginatedStageTurnList,
CommandLogResponse,
CommandOutputStream,
RunBilling,
@ -24,6 +23,7 @@ import {
apiNullableTextFetcher,
apiPaginatedFetcher,
apiTextFetcher,
fetchAllStageEvents,
type PaginatedEnvelope,
} from "./api-client";
import { queryKeys } from "./query-keys";
@ -34,6 +34,9 @@ const immutableOptions: SWRConfiguration = {
revalidateOnReconnect: false,
};
type BoardRunsEnvelope = PaginatedEnvelope<PaginatedBoardRunList["data"][number]> &
Pick<PaginatedBoardRunList, "columns">;
export function useAuthConfig() {
return useSWR<{ methods: string[] }>(queryKeys.auth.config(), apiFetcher, immutableOptions);
}
@ -61,11 +64,7 @@ export function useSystemInfo() {
}
export function useBoardsRuns() {
return useSWR<
PaginatedEnvelope<PaginatedBoardRunList["data"][number]> & {
columns: { id: string; name: string }[];
}
>(queryKeys.boards.runs(), apiPaginatedFetcher);
return useSWR<BoardRunsEnvelope>(queryKeys.boards.runs(), apiPaginatedFetcher);
}
export function useRun(id: string | undefined) {
@ -141,21 +140,10 @@ export function useRunQuestions(id: string | undefined, enabled: boolean) {
);
}
export function useRunStageTurns(
id: string | undefined,
stageId: string | undefined,
enabled = true,
) {
return useSWR<PaginatedStageTurnList | null>(
id && stageId && enabled ? queryKeys.runs.stageTurns(id, stageId) : null,
apiNullableFetcher,
);
}
export function useRunEventsList(id: string | undefined, enabled = true) {
return useSWR<PaginatedEventList | null>(
id && enabled ? queryKeys.runs.events(id, 1000) : null,
apiNullableFetcher,
export function useRunStageEvents(id: string | undefined, stageId: string | undefined) {
return useSWR<EventEnvelope[]>(
id && stageId ? queryKeys.runs.stageEvents(id, stageId) : null,
fetchAllStageEvents<EventEnvelope>,
);
}
@ -206,4 +194,4 @@ export function useServerSettings() {
return useSWR<ServerSettings>(queryKeys.settings.server(), apiFetcher, immutableOptions);
}
export { apiTextFetcher };
export { apiTextFetcher };

25
apps/fabro-web/app/lib/query-keys.test.ts
View file

@ -11,6 +11,9 @@ describe("queryKeys", () => {
expect(queryKeys.runs.stageLog("run 1", "build step@2", "stderr", 12, 34)).toBe(
"/api/v1/runs/run%201/stages/build%20step%402/logs/stderr?offset=12&limit=34",
);
expect(queryKeys.runs.stageEvents("run 1", "build step", 7, 25)).toBe(
"/api/v1/runs/run%201/stages/build%20step/events?since_seq=7&limit=25",
);
});
test("event-mapped keys match query hook resources", () => {
@ -19,11 +22,31 @@ describe("queryKeys", () => {
]);
expect(queryKeysForRunEvent("run-1", "stage.completed", "stage-1")).toEqual([
queryKeys.runs.stages("run-1"),
queryKeys.runs.billing("run-1"),
queryKeys.runs.events("run-1", 1000),
queryKeys.runs.graph("run-1", "LR"),
queryKeys.runs.graph("run-1", "TB"),
queryKeys.runs.detail("run-1"),
queryKeys.runs.stageTurns("run-1", "stage-1"),
queryKeys.runs.stageEvents("run-1", "stage-1"),
]);
});
test("agent activity events invalidate the per-stage events key", () => {
for (const event of [
"stage.prompt",
"agent.message",
"agent.tool.started",
"agent.tool.completed",
"command.started",
"command.completed",
]) {
expect(queryKeysForRunEvent("run-1", event, "stage-1")).toEqual([
queryKeys.runs.stageEvents("run-1", "stage-1"),
]);
}
});
test("agent activity events without a node_id invalidate nothing", () => {
expect(queryKeysForRunEvent("run-1", "agent.message")).toEqual([]);
});
});

9
apps/fabro-web/app/lib/query-keys.ts
View file

@ -44,8 +44,11 @@ export const queryKeys = {
}),
events: (id: string, limit = 1000) =>
withQuery(`/api/v1/runs/${pathSegment(id)}/events`, { limit }),
stageTurns: (id: string, stageId: string) =>
`/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/turns`,
stageEvents: (id: string, stageId: string, sinceSeq?: number, limit?: number) =>
withQuery(`/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/events`, {
since_seq: sinceSeq,
limit,
}),
stageLog: (
id: string,
stageId: string,
@ -75,4 +78,4 @@ export const queryKeys = {
settings: {
server: () => "/api/v1/settings",
},
};
};

201
apps/fabro-web/app/lib/run-events.test.tsx
View file

@ -4,7 +4,12 @@ import {
queryKeysForRunEvent,
subscribeToRunEvents,
} from "./run-events";
import {
createCrossTabSseCoordinator,
type BroadcastChannelLike,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import type { EventSourceLike } from "./sse";
type MessageHandler = ((event: { data: string }) => void) | null;
@ -25,6 +30,14 @@ class FakeEventSource {
}
}
class FakeBroadcastChannel implements BroadcastChannelLike {
onmessage: ((event: { data: unknown }) => void) | null = null;
postMessage() {}
close() {}
}
describe("queryKeysForRunEvent", () => {
test("terminal events invalidate run-scoped resources", () => {
expect(queryKeysForRunEvent("run-1", "run.completed")).toEqual([
@ -36,13 +49,91 @@ describe("queryKeysForRunEvent", () => {
queryKeys.runs.graph("run-1", "TB"),
]);
});
test("stage.retrying invalidates stages, billing, events, graph, detail, and stage events", () => {
expect(queryKeysForRunEvent("run-1", "stage.retrying", "verify@2")).toEqual([
queryKeys.runs.stages("run-1"),
queryKeys.runs.billing("run-1"),
queryKeys.runs.events("run-1", 1000),
queryKeys.runs.graph("run-1", "LR"),
queryKeys.runs.graph("run-1", "TB"),
queryKeys.runs.detail("run-1"),
queryKeys.runs.stageEvents("run-1", "verify@2"),
]);
});
});
describe("subscribeToRunEvents", () => {
test("refcounts shared sources and keeps mutators active until final unsubscribe", () => {
test("coordinated mode uses the global attach stream and filters by run_id", async () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createCoordinator((url) => {
created.push(url);
return source;
});
const cleanup = subscribeToRunEvents(
"run-coordinated",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => {
throw new Error("source should be created by coordinator");
},
{ debounceMs: 0, coordinator },
);
await waitFor(() => created.length === 1);
keys.length = 0;
source.emit({ event: "checkpoint.completed", run_id: "other-run" });
source.emit({ event: "checkpoint.completed", run_id: "run-coordinated" });
expect(created).toEqual(["/api/v1/attach"]);
expect(keys).toEqual([queryKeys.runs.files("run-coordinated")]);
cleanup();
coordinator.close();
});
test("coordinated terminal events invalidate without closing the global stream", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-terminal",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
keys.length = 0;
source.emit({ event: "run.failed", run_id: "run-terminal" });
expect(source.closed).toBe(false);
expect(keys).toContain(queryKeys.runs.files("run-terminal"));
expect(keys).toContain(queryKeys.runs.billing("run-terminal"));
keys.length = 0;
source.emit({ event: "run.archived", run_id: "run-terminal" });
expect(source.closed).toBe(false);
expect(keys).toEqual([queryKeys.runs.detail("run-terminal")]);
cleanup();
coordinator.close();
});
test("fallback refcounts run-scoped sources and keeps mutators active until final unsubscribe", () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const mutate = (key: string) => {
keys.push(key);
return Promise.resolve();
@ -51,10 +142,10 @@ describe("subscribeToRunEvents", () => {
const firstCleanup = subscribeToRunEvents("run-refcount", mutate, (url) => {
created.push(url);
return source;
}, { debounceMs: 0 });
}, { debounceMs: 0, coordinator });
const secondCleanup = subscribeToRunEvents("run-refcount", mutate, () => {
throw new Error("source should be reused");
}, { debounceMs: 0 });
}, { debounceMs: 0, coordinator });
expect(created).toEqual(["/api/v1/runs/run-refcount/attach"]);
@ -66,11 +157,13 @@ describe("subscribeToRunEvents", () => {
secondCleanup();
expect(source.closed).toBe(true);
coordinator.close();
});
test("terminal events close the source after invalidating keys", () => {
test("fallback terminal events close the source after invalidating keys", () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const cleanup = subscribeToRunEvents(
"run-terminal",
(key) => {
@ -78,7 +171,7 @@ describe("subscribeToRunEvents", () => {
return Promise.resolve();
},
() => source,
{ debounceMs: 0 },
{ debounceMs: 0, coordinator },
);
source.emit({ event: "run.failed" });
@ -88,13 +181,72 @@ describe("subscribeToRunEvents", () => {
expect(keys).toContain(queryKeys.runs.billing("run-terminal"));
cleanup();
coordinator.close();
});
test("malformed events are ignored and StrictMode-style cleanup does not underflow", () => {
test("envelope with suffixed stage_id invalidates stageEvents(runId, stageId)", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-stage",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
source.emit({
event: "stage.retrying",
run_id: "run-stage",
stage_id: "verify@2",
node_id: "verify",
});
expect(keys).toContain(queryKeys.runs.stageEvents("run-stage", "verify@2"));
expect(keys).toContain(queryKeys.runs.stages("run-stage"));
expect(keys).toContain(queryKeys.runs.events("run-stage", 1000));
expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR"));
expect(keys).toContain(queryKeys.runs.detail("run-stage"));
expect(keys).not.toContain(queryKeys.runs.stageEvents("run-stage", "verify"));
cleanup();
coordinator.close();
});
test("falls back to node_id when an event has no stage_id", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-stage-node",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" });
expect(keys).toContain(queryKeys.runs.stageEvents("run-stage-node", "verify"));
expect(keys).toContain(queryKeys.runs.stages("run-stage-node"));
cleanup();
coordinator.close();
});
test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => {
const firstSource = new FakeEventSource();
const secondSource = new FakeEventSource();
const sources = [firstSource, secondSource];
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const firstCleanup = subscribeToRunEvents(
"run-strict",
@ -103,7 +255,7 @@ describe("subscribeToRunEvents", () => {
return Promise.resolve();
},
() => sources.shift()!,
{ debounceMs: 0 },
{ debounceMs: 0, coordinator },
);
firstSource.emitRaw("{broken");
firstCleanup();
@ -115,12 +267,45 @@ describe("subscribeToRunEvents", () => {
return Promise.resolve();
},
() => sources.shift()!,
{ debounceMs: 0 },
{ debounceMs: 0, coordinator },
);
secondCleanup();
expect(keys).toEqual([]);
expect(firstSource.closed).toBe(true);
expect(secondSource.closed).toBe(true);
coordinator.close();
});
});
function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) {
return createCrossTabSseCoordinator({
tabId: "run-test",
channelFactory: () => new FakeBroadcastChannel(),
eventSourceFactory,
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: {
heartbeatMs: 10,
leaderStaleMs: 50,
electionJitterMs: 0,
},
});
}
function createFallbackCoordinator() {
return createCrossTabSseCoordinator({
channelFactory: () => {
throw new Error("BroadcastChannel unavailable");
},
});
}
async function waitFor(condition: () => boolean, timeoutMs = 200) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (condition()) return;
await new Promise((resolve) => setTimeout(resolve, 2));
}
throw new Error("condition did not become true before timeout");
}

113
apps/fabro-web/app/lib/run-events.ts
View file

@ -1,6 +1,10 @@
import { useEffect } from "react";
import { useSWRConfig } from "swr";
import {
subscribeToCrossTabSse,
type CrossTabSseCoordinator,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import {
createBrowserEventSource,
@ -13,10 +17,17 @@ import {
interface RunEventPayload extends EventPayload {
event?: string;
run_id?: string;
node_id?: string;
stage_id?: string;
properties?: Record<string, unknown>;
}
interface RunEventOptions {
debounceMs?: number;
coordinator?: CrossTabSseCoordinator;
}
const subscriptions = new Map<string, SharedEventSubscription>();
const TERMINAL_EVENTS = new Set(["run.completed", "run.failed"]);
@ -32,8 +43,31 @@ const RUN_SUMMARY_EVENTS = new Set([
"run.archived",
"run.unarchived",
]);
const STAGE_EVENTS = new Set(["stage.started", "stage.completed", "stage.failed"]);
const COMMAND_EVENTS = new Set(["command.started", "command.completed"]);
const STAGE_EVENTS = new Set([
"stage.started",
"stage.completed",
"stage.failed",
"stage.retrying",
]);
// Single source of truth: every event type the `eventsToActivity` reducer in
// `routes/run-stages.tsx` consumes. When any of these arrive for a stage we
// currently view, the stage-events SWR key for that stage must be invalidated
// so the panel refetches. The reducer imports this list so the switch stays
// in sync with the invalidation set; if the reducer grows a new case, this
// list is the single edit point.
//
// The lifecycle `STAGE_EVENTS` set is kept separate because it also fans out
// to run-scoped invalidations (stages list, graph, detail).
export const STAGE_ACTIVITY_EVENT_TYPES = [
"stage.prompt",
"agent.message",
"agent.tool.started",
"agent.tool.completed",
"command.started",
"command.completed",
] as const;
export type StageActivityEventType = (typeof STAGE_ACTIVITY_EVENT_TYPES)[number];
const STAGE_ACTIVITY_EVENTS = new Set<string>(STAGE_ACTIVITY_EVENT_TYPES);
const INTERVIEW_EVENTS = new Set([
"interview.started",
"interview.completed",
@ -75,26 +109,20 @@ export function queryKeysForRunEvent(
if (STAGE_EVENTS.has(event)) {
const keys = [
queryKeys.runs.stages(runId),
queryKeys.runs.billing(runId),
queryKeys.runs.events(runId, 1000),
queryKeys.runs.graph(runId, "LR"),
queryKeys.runs.graph(runId, "TB"),
queryKeys.runs.detail(runId),
];
if (stageId) {
keys.push(queryKeys.runs.stageTurns(runId, stageId));
keys.push(queryKeys.runs.stageEvents(runId, stageId));
}
return keys;
}
if (COMMAND_EVENTS.has(event)) {
const keys = [
queryKeys.runs.stages(runId),
queryKeys.runs.events(runId, 1000),
];
if (stageId) {
keys.push(queryKeys.runs.stageTurns(runId, stageId));
}
return keys;
if (STAGE_ACTIVITY_EVENTS.has(event)) {
return stageId ? [queryKeys.runs.stageEvents(runId, stageId)] : [];
}
return [];
@ -104,32 +132,59 @@ export function subscribeToRunEvents(
runId: string,
mutate: MutateFn,
eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource,
{ debounceMs = 300 }: { debounceMs?: number } = {},
{ debounceMs = 300, coordinator }: RunEventOptions = {},
): () => void {
return subscribeToSharedEventSource<RunEventPayload>({
subscriptions,
subscriptionKey: runId,
url: queryKeys.runs.attach(runId),
return subscribeToCrossTabSse<RunEventPayload>({
coordinator,
subscriptionKey: `run:${runId}`,
mutate,
eventSourceFactory,
debounceMs,
resyncKeys: () => resyncKeysForRun(runId),
resolveInvalidation: (payload) => {
const event = payload.event;
if (!event) return { keys: [] };
const stageId = stageIdFromPayload(payload);
const keys = queryKeysForRunEvent(runId, event, stageId);
const terminal = TERMINAL_EVENTS.has(event);
return {
keys,
close: terminal,
immediate: terminal,
};
if (payload.run_id !== runId) return { keys: [] };
return runInvalidation(runId, payload);
},
fallbackSubscribe: () =>
subscribeToSharedEventSource<RunEventPayload>({
subscriptions,
subscriptionKey: runId,
url: queryKeys.runs.attach(runId),
mutate,
eventSourceFactory,
debounceMs,
resolveInvalidation: (payload) => {
const result = runInvalidation(runId, payload);
return { ...result, close: result.immediate };
},
}),
});
}
function runInvalidation(runId: string, payload: RunEventPayload) {
const event = payload.event;
if (!event) return { keys: [], immediate: false };
const stageId = stageIdFromPayload(payload);
const keys = queryKeysForRunEvent(runId, event, stageId);
const terminal = TERMINAL_EVENTS.has(event);
return { keys, immediate: terminal };
}
function resyncKeysForRun(runId: string) {
return [
queryKeys.runs.detail(runId),
queryKeys.runs.files(runId),
queryKeys.runs.billing(runId),
queryKeys.runs.stages(runId),
queryKeys.runs.events(runId, 1000),
queryKeys.runs.graph(runId, "LR"),
queryKeys.runs.graph(runId, "TB"),
queryKeys.runs.questions(runId, 25, 0),
];
}
function stageIdFromPayload(payload: RunEventPayload): string | undefined {
if (typeof payload.stage_id === "string") return payload.stage_id;
if (typeof payload.node_id === "string") return payload.node_id;
const nodeId = payload.properties?.node_id;
return typeof nodeId === "string" ? nodeId : undefined;

172
apps/fabro-web/app/lib/stage-sidebar.test.ts Normal file
View file

@ -0,0 +1,172 @@
import { describe, expect, test } from "bun:test";
import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client";
import type { Stage } from "../components/stage-sidebar";
import { aggregateGraphNodeStatus, formatStageLabel, mapRunStagesToSidebarStages } from "./stage-sidebar";
function makeStage(nodeId: string, visit: number, status: StageState): Stage {
return {
id: `${nodeId}@${visit}`,
name: nodeId,
nodeId,
visit,
status,
duration: "--",
};
}
describe("mapRunStagesToSidebarStages", () => {
test("maps two visits of the same node to distinct sidebar entries", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "apply-changes@1",
name: "Apply Changes",
status: "succeeded",
duration_secs: 12.5,
node_id: "apply",
visit: 1,
},
{
id: "apply-changes@2",
name: "Apply Changes",
status: "running",
node_id: "apply",
visit: 2,
},
],
meta: { has_more: false },
};
const result = mapRunStagesToSidebarStages(stages);
expect(result).toHaveLength(2);
expect(result[0].id).toBe("apply-changes@1");
expect(result[0].nodeId).toBe("apply");
expect(result[0].visit).toBe(1);
expect(formatStageLabel(result[0])).toBe("Apply Changes");
expect(result[1].id).toBe("apply-changes@2");
expect(result[1].nodeId).toBe("apply");
expect(result[1].visit).toBe(2);
expect(formatStageLabel(result[1])).toBe("Apply Changes (2)");
});
test("filters by node_id (suffixed start@1 / exit@1 are still hidden)", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "start@1",
name: "start",
status: "succeeded",
node_id: "start",
visit: 1,
},
{
id: "verify@1",
name: "verify",
status: "succeeded",
node_id: "verify",
visit: 1,
},
{
id: "exit@1",
name: "exit",
status: "succeeded",
node_id: "exit",
visit: 1,
},
],
meta: { has_more: false },
};
const result = mapRunStagesToSidebarStages(stages);
expect(result.map((s) => s.id)).toEqual(["verify@1"]);
});
test("missing duration renders as '--'", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "verify@1",
name: "verify",
status: "running",
node_id: "verify",
visit: 1,
},
],
meta: { has_more: false },
};
expect(mapRunStagesToSidebarStages(stages)[0].duration).toBe("--");
});
});
describe("aggregateGraphNodeStatus", () => {
test("(failed, running) renders as running and clicks open the latest visit", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "failed"),
makeStage("verify", 2, "running"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "running",
latestStageId: "verify@2",
});
});
test("(failed, succeeded) renders as succeeded — failure-then-fix shows healed", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "failed"),
makeStage("verify", 2, "succeeded"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "succeeded",
latestStageId: "verify@2",
});
});
test("(succeeded, failed) renders as failed and clicks open the latest visit", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "succeeded"),
makeStage("verify", 2, "failed"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "failed",
latestStageId: "verify@2",
});
});
test("(running, retrying) — latest active wins", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "running"),
makeStage("verify", 2, "retrying"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "retrying",
latestStageId: "verify@2",
});
});
test("orders by visit even when input is shuffled", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 2, "running"),
makeStage("verify", 1, "failed"),
]);
expect(result.get("verify")?.latestStageId).toBe("verify@2");
});
test("single visit per node is unaffected", () => {
const result = aggregateGraphNodeStatus([
makeStage("plan", 1, "succeeded"),
makeStage("apply", 1, "running"),
]);
expect(result.get("plan")).toEqual({
displayStatus: "succeeded",
latestStageId: "plan@1",
});
expect(result.get("apply")).toEqual({
displayStatus: "running",
latestStageId: "apply@1",
});
});
});

70
apps/fabro-web/app/lib/stage-sidebar.ts
View file

@ -1,27 +1,83 @@
import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client";
import { StageState } from "@qltysh/fabro-api-client";
import type { PaginatedRunStageList } from "@qltysh/fabro-api-client";
import type { Stage } from "../components/stage-sidebar";
import { isVisibleStage } from "../data/runs";
import { formatDurationSecs } from "./format";
export const ACTIVE_STAGE_STATES: ReadonlySet<StageState> = new Set(["running", "retrying"]);
export const SUCCEEDED_STAGE_STATES: ReadonlySet<StageState> = new Set([
"succeeded",
"partially_succeeded",
export const ACTIVE_STAGE_STATES: ReadonlySet<StageState> = new Set([
StageState.RUNNING,
StageState.RETRYING,
]);
export const IN_FLIGHT_STAGE_STATES: ReadonlySet<StageState> = new Set([
StageState.PENDING,
StageState.RUNNING,
StageState.RETRYING,
]);
export const SUCCEEDED_STAGE_STATES: ReadonlySet<StageState> = new Set([
StageState.SUCCEEDED,
StageState.PARTIALLY_SUCCEEDED,
]);
/**
* Display label for a stage. Suffixes `(N)` for visits > 1 so a looped node
* (e.g. `verify`) renders as `verify`, `verify (2)`, `verify (3)` in the
* sidebar and stage header.
*/
export function formatStageLabel(stage: { name: string; visit: number }): string {
return stage.visit > 1 ? `${stage.name} (${stage.visit})` : stage.name;
}
export function mapRunStagesToSidebarStages(
stagesResult: PaginatedRunStageList | null | undefined,
): Stage[] {
return (stagesResult?.data ?? [])
.filter((stage) => isVisibleStage(stage.id))
.filter((stage) => isVisibleStage(stage.node_id))
.map((stage) => ({
id: stage.id,
name: stage.name,
dotId: stage.dot_id ?? stage.id,
nodeId: stage.node_id,
visit: stage.visit,
status: stage.status,
duration: stage.duration_secs != null
? formatDurationSecs(stage.duration_secs)
: "--",
}));
}
/**
* Aggregate per-node display state for the workflow graph.
*
* Status policy: if any visit is active (running/retrying), the node renders
* that active state (latest active visit wins). Otherwise the node renders
* the latest visit's terminal state. The click target is always the latest
* visit's stageId.
*/
export function aggregateGraphNodeStatus(stages: readonly Stage[]): Map<
string,
{ displayStatus: StageState; latestStageId: string }
> {
// Single pass per nodeId: track the visit with the highest `visit` overall
// (drives click target + terminal status) and the highest-visit *active*
// stage (drives display when any visit is in flight).
const latest = new Map<string, Stage>();
const latestActive = new Map<string, Stage>();
for (const stage of stages) {
const prevLatest = latest.get(stage.nodeId);
if (!prevLatest || stage.visit > prevLatest.visit) {
latest.set(stage.nodeId, stage);
}
if (ACTIVE_STAGE_STATES.has(stage.status)) {
const prevActive = latestActive.get(stage.nodeId);
if (!prevActive || stage.visit > prevActive.visit) {
latestActive.set(stage.nodeId, stage);
}
}
}
const result = new Map<string, { displayStatus: StageState; latestStageId: string }>();
for (const [nodeId, latestStage] of latest) {
const display = latestActive.get(nodeId) ?? latestStage;
result.set(nodeId, { displayStatus: display.status, latestStageId: latestStage.id });
}
return result;
}

19
apps/fabro-web/app/lib/time.ts
View file

@ -1,3 +1,21 @@
import { useEffect, useState } from "react";
/**
* Re-renders the calling component every `intervalMs` milliseconds while
* `active` is true, returning the current `Date.now()` value at each tick.
* Returns the captured value when paused, so renders are stable.
*/
export function useTickingNow(active: boolean, intervalMs = 1000): number {
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!active) return;
setNow(Date.now());
const interval = setInterval(() => setNow(Date.now()), intervalMs);
return () => clearInterval(interval);
}, [active, intervalMs]);
return now;
}
function relativeTime(seconds: number, past: boolean): string {
if (seconds < 60) return past ? "just now" : "in <1m";
const minutes = Math.floor(seconds / 60);
@ -21,4 +39,3 @@ export function timeAgo(iso: string): string {
export function timeUntil(iso: string): string {
return relativeTime(Math.floor((new Date(iso).getTime() - Date.now()) / 1000), false);
}

62
apps/fabro-web/app/routes/run-billing.test.tsx
View file

@ -75,12 +75,14 @@ describe("RunBilling", () => {
model: null,
billing: zeroBilling(),
runtime_secs: 0,
state: "succeeded",
},
{
stage: { id: "command", name: "command" },
model: null,
billing: zeroBilling(),
runtime_secs: 61,
state: "succeeded",
},
],
totals: {
@ -96,7 +98,7 @@ describe("RunBilling", () => {
expect(text).toMatch(/—\s*\/\s*—/);
expect(text).toContain("1m 1s");
expect(text).not.toContain("By model");
expect(text).not.toContain("No completed stages yet");
expect(text).not.toContain("No stages yet");
});
test("renders mixed LLM and non-LLM rows while counting only LLM rows by model", () => {
@ -108,6 +110,7 @@ describe("RunBilling", () => {
model: null,
billing: zeroBilling(),
runtime_secs: 0,
state: "succeeded",
},
{
stage: { id: "agent", name: "agent" },
@ -119,6 +122,7 @@ describe("RunBilling", () => {
total_usd_micros: 240000,
}),
runtime_secs: 42,
state: "succeeded",
},
],
totals: {
@ -155,11 +159,61 @@ describe("RunBilling", () => {
expect(textFromInstance(byModelFooterCells[1])).toBe("1");
});
test("keeps the empty state for runs with no completed stages", () => {
test("keeps the empty state for runs with no stages", () => {
const renderer = renderBilling(billing());
const text = textFromNode(renderer.toJSON());
expect(text).toContain("No completed stages yet");
expect(text).toContain("Stages will appear once the run produces completed nodes.");
expect(text).toContain("No stages yet");
expect(text).toContain("Stages will appear as soon as the run starts executing.");
});
test("renders an in-flight row with live runtime and includes its elapsed time in the footer", () => {
const originalNow = Date.now;
// Pin "now" to 30s after the in-flight row started.
const startedAt = "2026-04-29T12:00:00.000Z";
const fakeNow = new Date("2026-04-29T12:00:30.000Z").getTime();
Date.now = () => fakeNow;
try {
const renderer = renderBilling(
billing({
stages: [
{
stage: { id: "in-flight", name: "in-flight" },
model: null,
// Server reports 0 runtime / no billing; the row is still being executed.
billing: zeroBilling(),
runtime_secs: 0,
started_at: startedAt,
state: "running",
},
],
// Server total is 0 because the in-flight row hasn't been finalized.
totals: {
runtime_secs: 0,
...zeroBilling(),
},
}),
);
const text = textFromNode(renderer.toJSON());
// Empty-state must NOT show — the table should appear as soon as the
// first stage starts.
expect(text).not.toContain("No stages yet");
expect(text).toContain("in-flight");
// Both the row's runtime cell and the footer total should reflect
// ~30s elapsed since started_at.
expect(text).toContain("30s");
const footers = renderer.root.findAll((node) => node.type === "tfoot");
const footerCells = footers[0].findAll((node) => node.type === "td");
// The Run time column in the footer is index 3 (Total / [empty Model] /
// Tokens / Run time / Billing).
const footerRuntime = textFromInstance(footerCells[3]);
expect(footerRuntime).toContain("30s");
} finally {
Date.now = originalNow;
}
});
});

161
apps/fabro-web/app/routes/run-billing.tsx
View file

@ -1,7 +1,11 @@
import { useMemo } from "react";
import { EmptyState } from "../components/state";
import { formatDurationSecs } from "../lib/format";
import { useRunBilling } from "../lib/queries";
import type { RunBilling } from "@qltysh/fabro-api-client";
import { IN_FLIGHT_STAGE_STATES } from "../lib/stage-sidebar";
import { useTickingNow } from "../lib/time";
import type { RunBilling, RunBillingStage } from "@qltysh/fabro-api-client";
const EMPTY_VALUE = "—";
@ -14,78 +18,103 @@ function formatUsdMicros(usdMicros?: number | null) {
return usdMicros == null ? EMPTY_VALUE : `$${(usdMicros / 1_000_000).toFixed(2)}`;
}
function mapBilling(billing: RunBilling | undefined) {
if (!billing) {
return {
stages: [],
totalRuntime: formatDurationSecs(0),
totalUsdMicros: undefined,
totalInput: null,
totalOutput: null,
modelBreakdown: [],
modelStageCount: 0,
};
}
function isInFlight(stage: RunBillingStage): boolean {
return stage.state != null && IN_FLIGHT_STAGE_STATES.has(stage.state);
}
const stages = billing.stages.map((stage) => {
const hasModel = stage.model != null;
return {
stage: stage.stage.name,
model: stage.model?.id ?? null,
inputTokens: hasModel ? stage.billing.input_tokens : null,
outputTokens: hasModel
? stage.billing.output_tokens + stage.billing.reasoning_tokens
: null,
runtime: formatDurationSecs(stage.runtime_secs),
totalUsdMicros: stage.billing.total_usd_micros,
};
});
const totalRuntime = formatDurationSecs(billing.totals.runtime_secs);
const hasLlmStages = billing.by_model.length > 0;
const totalInput = hasLlmStages ? billing.totals.input_tokens : null;
const totalOutput = hasLlmStages
? billing.totals.output_tokens + billing.totals.reasoning_tokens
: null;
const totalUsdMicros = billing.totals.total_usd_micros;
const modelBreakdown = billing.by_model
.map((entry) => ({
model: entry.model.id,
stages: entry.stages,
inputTokens: entry.billing.input_tokens,
outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens,
totalUsdMicros: entry.billing.total_usd_micros,
}))
.sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1));
const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0);
interface MappedStageRow {
stage: string;
model: string | null;
inputTokens: number | null;
outputTokens: number | null;
runtimeSecs: number;
totalUsdMicros: number | null | undefined;
}
function liveRuntimeSecs(stage: RunBillingStage, now: number): number {
if (stage.started_at) {
const startedMs = new Date(stage.started_at).getTime();
if (Number.isFinite(startedMs)) {
return Math.max(0, (now - startedMs) / 1000);
}
}
return stage.runtime_secs;
}
function mapStageRow(stage: RunBillingStage, runtimeSecs: number): MappedStageRow {
const hasModel = stage.model != null;
return {
stages,
totalRuntime,
totalUsdMicros,
totalInput,
totalOutput,
modelBreakdown,
modelStageCount,
stage: stage.stage.name,
model: stage.model?.id ?? null,
inputTokens: hasModel ? stage.billing.input_tokens : null,
outputTokens: hasModel
? stage.billing.output_tokens + stage.billing.reasoning_tokens
: null,
runtimeSecs,
totalUsdMicros: stage.billing.total_usd_micros,
};
}
export default function RunBilling({ params }: { params: { id: string } }) {
const billingQuery = useRunBilling(params.id);
const {
stages,
totalRuntime,
totalUsdMicros,
totalInput,
totalOutput,
modelBreakdown,
modelStageCount,
} = mapBilling(billingQuery.data);
const billing = billingQuery.data;
const hasInFlight = billing?.stages.some(isInFlight) ?? false;
if (!stages.length) {
// Tick once per second only while a stage is in-flight.
const now = useTickingNow(hasInFlight);
// Completed rows don't depend on `now`; memoize them by `billing` so we
// don't reallocate them every tick.
const completedRows = useMemo<MappedStageRow[]>(() => {
if (!billing) return [];
return billing.stages.map((stage) => mapStageRow(stage, stage.runtime_secs));
}, [billing]);
// The model breakdown is server-derived and stable across ticks too.
const modelBreakdown = useMemo(() => {
if (!billing) return [];
return billing.by_model
.map((entry) => ({
model: entry.model.id,
stages: entry.stages,
inputTokens: entry.billing.input_tokens,
outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens,
totalUsdMicros: entry.billing.total_usd_micros,
}))
.sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1));
}, [billing]);
// Re-derive only the in-flight rows on each tick; everything else stays put.
const rows = useMemo<MappedStageRow[]>(() => {
if (!billing) return [];
if (!hasInFlight) return completedRows;
return billing.stages.map((stage, idx) =>
isInFlight(stage)
? mapStageRow(stage, liveRuntimeSecs(stage, now))
: completedRows[idx],
);
}, [billing, completedRows, hasInFlight, now]);
// While ticking, sum the displayed row runtimes so the footer updates in
// lock-step. Otherwise trust the server's authoritative total.
const totalRuntimeSecs = hasInFlight
? rows.reduce((sum, row) => sum + row.runtimeSecs, 0)
: (billing?.totals.runtime_secs ?? 0);
const hasLlmStages = (billing?.by_model.length ?? 0) > 0;
const totalInput = hasLlmStages ? (billing?.totals.input_tokens ?? null) : null;
const totalOutput = hasLlmStages && billing
? billing.totals.output_tokens + billing.totals.reasoning_tokens
: null;
const totalUsdMicros = billing?.totals.total_usd_micros;
const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0);
if (!rows.length) {
return (
<div className="py-12">
<EmptyState
title="No completed stages yet"
description="Stages will appear once the run produces completed nodes."
title="No stages yet"
description="Stages will appear as soon as the run starts executing."
/>
</div>
);
@ -105,7 +134,7 @@ export default function RunBilling({ params }: { params: { id: string } }) {
</tr>
</thead>
<tbody>
{stages.map((row) => (
{rows.map((row) => (
<tr key={row.stage} className="border-b border-line last:border-b-0">
<td className="px-4 py-3 text-fg-2">{row.stage}</td>
<td className="px-4 py-3 font-mono text-xs text-fg-3">
@ -115,7 +144,9 @@ export default function RunBilling({ params }: { params: { id: string } }) {
{formatTokens(row.inputTokens)} <span className="text-fg-muted">/</span>{" "}
{formatTokens(row.outputTokens)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs text-fg-3">{row.runtime}</td>
<td className="px-4 py-3 text-right font-mono text-xs text-fg-3">
{formatDurationSecs(row.runtimeSecs)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs text-fg-3">
{formatUsdMicros(row.totalUsdMicros)}
</td>
@ -131,7 +162,7 @@ export default function RunBilling({ params }: { params: { id: string } }) {
{formatTokens(totalOutput)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs font-medium text-fg">
{totalRuntime}
{formatDurationSecs(totalRuntimeSecs)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs font-medium text-fg">
{formatUsdMicros(totalUsdMicros)}

34
apps/fabro-web/app/routes/run-overview.tsx
View file

@ -3,7 +3,6 @@ import { useNavigate, useParams } from "react-router";
import { graphTheme } from "../lib/graph-theme";
import { useRun, useRunGraph, useRunStages } from "../lib/queries";
import { StageSidebar } from "../components/stage-sidebar";
import type { Stage } from "../components/stage-sidebar";
import {
GRAPH_DEFAULT_ZOOM_INDEX,
GRAPH_ZOOM_STEPS,
@ -13,6 +12,7 @@ import { EmptyState } from "../components/state";
import {
ACTIVE_STAGE_STATES,
SUCCEEDED_STAGE_STATES,
aggregateGraphNodeStatus,
mapRunStagesToSidebarStages,
} from "../lib/stage-sidebar";
@ -54,27 +54,27 @@ export default function RunOverview() {
const inner = innerRef.current;
if (!inner || !graphSvg) return;
let cancelled = false;
(async () => {
if (cancelled) return;
inner.innerHTML = graphSvg;
const svg = inner.querySelector("svg");
if (!svg) return;
svgRef.current = svg;
const gt = graphTheme;
const runningDotIds = new Set<string>(
stages.filter((s: Stage) => ACTIVE_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id),
);
const failedDotIds = new Set<string>(
stages.filter((s: Stage) => s.status === "failed").map((s: Stage) => s.dotId ?? s.id),
);
const completedDotIds = new Set<string>(
stages.filter((s: Stage) => SUCCEEDED_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id),
);
const dotIdToStageId = new Map<string, string>(
stages.map((s: Stage) => [s.dotId ?? s.id, s.id]),
);
const aggregated = aggregateGraphNodeStatus(stages);
const runningDotIds = new Set<string>();
const failedDotIds = new Set<string>();
const completedDotIds = new Set<string>();
const dotIdToStageId = new Map<string, string>();
for (const [nodeId, { displayStatus, latestStageId }] of aggregated) {
dotIdToStageId.set(nodeId, latestStageId);
if (ACTIVE_STAGE_STATES.has(displayStatus)) {
runningDotIds.add(nodeId);
} else if (displayStatus === "failed") {
failedDotIds.add(nodeId);
} else if (SUCCEEDED_STAGE_STATES.has(displayStatus)) {
completedDotIds.add(nodeId);
}
}
const ns = "http://www.w3.org/2000/svg";
for (const group of svg.querySelectorAll(".node")) {
@ -148,8 +148,6 @@ export default function RunOverview() {
}
}
}
})();
return () => { cancelled = true; };
}, [stages, graphSvg, id, navigate, terminalOutcome]);
const onPointerDown = useCallback((e: React.PointerEvent) => {

191
apps/fabro-web/app/routes/run-stages.test.ts
View file

@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import type { EventEnvelope } from "@qltysh/fabro-api-client";
import { isSafeMarkdownHref } from "./run-stages";
import { eventsToActivity, isSafeMarkdownHref } from "./run-stages";
describe("isSafeMarkdownHref", () => {
test("rejects protocol-relative URLs", () => {
@ -15,3 +16,191 @@ describe("isSafeMarkdownHref", () => {
expect(isSafeMarkdownHref("mailto:test@example.com")).toBe(true);
});
});
function envelope(seq: number, partial: Partial<EventEnvelope>): EventEnvelope {
return {
seq,
id: `evt-${seq}`,
ts: "2026-04-09T12:00:00Z",
run_id: "run-1",
event: "stage.prompt",
...partial,
} as EventEnvelope;
}
describe("eventsToActivity", () => {
test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "stage.prompt",
stage_id: "verify@1",
node_id: "verify",
properties: { text: "first visit prompt" },
}),
envelope(2, {
event: "stage.prompt",
stage_id: "verify@2",
node_id: "verify",
properties: { text: "second visit prompt" },
}),
envelope(3, {
event: "agent.message",
stage_id: "verify@1",
node_id: "verify",
properties: { text: "first visit reply" },
}),
envelope(4, {
event: "agent.message",
stage_id: "verify@2",
node_id: "verify",
properties: { text: "second visit reply" },
}),
];
const firstVisit = eventsToActivity(events, "verify@1");
expect(firstVisit).toEqual([
{ kind: "system", content: "first visit prompt" },
{ kind: "assistant", content: "first visit reply" },
]);
const secondVisit = eventsToActivity(events, "verify@2");
expect(secondVisit).toEqual([
{ kind: "system", content: "second visit prompt" },
{ kind: "assistant", content: "second visit reply" },
]);
});
test("pairs command.started + command.completed into a single command turn", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "command.started",
node_id: "fmt",
properties: { script: "cargo fmt", language: "shell" },
}),
envelope(2, {
event: "command.completed",
node_id: "fmt",
properties: {
stdout: "ok",
stderr: "",
exit_code: 0,
duration_ms: 12,
termination: "exited",
},
}),
];
const turns = eventsToActivity(events, "fmt");
expect(turns).toHaveLength(1);
expect(turns[0]).toMatchObject({
kind: "command",
stageId: "fmt",
script: "cargo fmt",
language: "shell",
stdout: "ok",
exitCode: 0,
running: false,
});
});
test("command turn carries the requested stage_id, no @1 fallback", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "command.started",
stage_id: "verify@2",
node_id: "verify",
properties: { script: "echo hi", language: "shell" },
}),
envelope(2, {
event: "command.completed",
stage_id: "verify@2",
node_id: "verify",
properties: {
stdout: "hi",
stderr: "",
exit_code: 0,
duration_ms: 5,
termination: "exited",
},
}),
];
const turns = eventsToActivity(events, "verify@2");
expect(turns).toHaveLength(1);
const turn = turns[0];
expect(turn.kind).toBe("command");
if (turn.kind === "command") {
expect(turn.stageId).toBe("verify@2");
expect(turn.script).toBe("echo hi");
expect(turn.running).toBe(false);
}
});
test("pairs agent.tool.started + agent.tool.completed into a single tool turn", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "agent.tool.started",
node_id: "detect-drift",
properties: {
tool_call_id: "call-1",
tool_name: "read_file",
arguments: { path: "config.toml" },
},
}),
envelope(2, {
event: "agent.tool.completed",
node_id: "detect-drift",
properties: {
tool_call_id: "call-1",
tool_name: "read_file",
output: "[redis]",
is_error: false,
},
}),
];
const turns = eventsToActivity(events, "detect-drift");
expect(turns).toHaveLength(1);
expect(turns[0].kind).toBe("tool");
if (turns[0].kind === "tool") {
expect(turns[0].tools).toHaveLength(1);
expect(turns[0].tools[0]).toMatchObject({
id: "call-1",
toolName: "read_file",
result: "[redis]",
isError: false,
});
}
});
test("ignores unknown event types and events for other stages", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "stage.started",
node_id: "detect-drift",
properties: {},
}),
envelope(2, {
event: "agent.message",
node_id: "detect-drift",
properties: { text: "signal" },
}),
envelope(3, {
event: "run.running",
node_id: "detect-drift",
properties: {},
}),
envelope(4, {
event: "agent.message",
node_id: "other-stage",
properties: { text: "wrong stage" },
}),
];
const turns = eventsToActivity(events, "detect-drift");
expect(turns).toHaveLength(1);
if (turns[0].kind === "assistant") {
expect(turns[0].content).toBe("signal");
}
});
});

114
apps/fabro-web/app/routes/run-stages.tsx
View file

@ -40,16 +40,15 @@ import type { Stage } from "../components/stage-sidebar";
import { EmptyState } from "../components/state";
import { CopyButton } from "../components/ui";
import { formatDurationSecs } from "../lib/format";
import { fetchRunCommandLog, useRunEventsList, useRunStageTurns, useRunStages } from "../lib/queries";
import { mapRunStagesToSidebarStages } from "../lib/stage-sidebar";
import { useTickingNow } from "../lib/time";
import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries";
import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events";
import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar";
import { getNumber, getString, type UnknownRecord } from "../lib/unknown";
import {
CommandOutputStream,
CommandTermination,
type EventEnvelope,
type StageTurn as ApiStageTurn,
type PaginatedStageTurnList,
type PaginatedEventList,
} from "@qltysh/fabro-api-client";
export const handle = { wide: true };
@ -68,17 +67,40 @@ function readTermination(props: UnknownRecord): CommandTermination {
return CommandTermination.EXITED;
}
function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
const stageEvents = events.filter((e) => e.node_id === stageId);
const STAGE_ACTIVITY_EVENT_SET = new Set<string>(STAGE_ACTIVITY_EVENT_TYPES);
function assertNever(value: never): never {
throw new Error(`Unhandled stage activity event type: ${value}`);
}
function activityEventStageId(event: EventEnvelope): string | undefined {
if (typeof event.stage_id === "string") return event.stage_id;
if (typeof event.node_id === "string") return event.node_id;
return getString(event.properties ?? {}, "node_id");
}
export function eventsToActivity(events: EventEnvelope[], stageId: string): TurnType[] {
const turns: TurnType[] = [];
// Collect tool pairs: started → completed
const pendingTools = new Map<string, { toolName: string; input: string }>();
// Track pending command for pairing started → completed
let pendingCommand: { stageId: string; script: string; language: string } | undefined;
for (const e of stageEvents) {
for (const e of events) {
const eventName = e.event;
if (
activityEventStageId(e) !== stageId ||
!eventName ||
!STAGE_ACTIVITY_EVENT_SET.has(eventName)
) {
continue;
}
// Exhaustive switch over StageActivityEventType: adding a new variant to
// STAGE_ACTIVITY_EVENT_TYPES forces a TS error here until the case is
// handled, keeping the SWR invalidation set and the reducer in sync.
const eventType = eventName as StageActivityEventType;
const props = e.properties ?? {};
switch (e.event) {
switch (eventType) {
case "stage.prompt":
turns.push({ kind: "system", content: getString(props, "text") ?? e.text ?? "" });
break;
@ -114,7 +136,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
}
case "command.started": {
pendingCommand = {
stageId: e.stage_id ?? `${stageId}@1`,
stageId,
script: getString(props, "script") ?? "",
language: getString(props, "language") ?? "shell",
};
@ -123,7 +145,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
case "command.completed": {
turns.push({
kind: "command",
stageId: pendingCommand?.stageId ?? e.stage_id ?? `${stageId}@1`,
stageId: pendingCommand?.stageId ?? stageId,
script: pendingCommand?.script ?? "",
language: pendingCommand?.language ?? "shell",
stdout: getString(props, "stdout") ?? "",
@ -136,6 +158,8 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
pendingCommand = undefined;
break;
}
default:
assertNever(eventType);
}
}
@ -153,41 +177,6 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
return turns;
}
function mapApiStageTurn(t: ApiStageTurn): TurnType {
switch (t.kind) {
case "tool":
return {
kind: "tool",
tools: (t.tools ?? []).map((tu) => ({
id: tu.id,
toolName: tu.tool_name,
input: tu.input,
result: tu.result,
isError: tu.is_error,
durationMs: tu.duration_ms,
})),
};
case "system":
case "assistant":
return { kind: t.kind, content: t.content ?? "" };
}
}
function mapTurns(
turnsResult: PaginatedStageTurnList | null | undefined,
eventsResult: PaginatedEventList | null | undefined,
selectedStageId: string | undefined,
): TurnType[] {
if (!selectedStageId) return [];
if (turnsResult?.data?.length) {
return turnsResult.data.map(mapApiStageTurn);
}
if (eventsResult?.data) {
return turnsFromEvents(eventsResult.data, selectedStageId);
}
return [];
}
function Markdown({ content }: { content: string }) {
const html = useMemo(() => markedSafe.parse(content, { async: false }) as string, [content]);
return (
@ -575,7 +564,6 @@ function RunningStageDuration({
const [startedAt, setStartedAt] = useState<number | null>(() =>
isRunning ? Date.now() : null,
);
const [, setTick] = useState(0);
useEffect(() => {
setStartedAt((current) => {
@ -584,14 +572,10 @@ function RunningStageDuration({
});
}, [isRunning]);
useEffect(() => {
if (!isRunning) return;
const interval = setInterval(() => setTick((tick) => tick + 1), 1000);
return () => clearInterval(interval);
}, [isRunning]);
const now = useTickingNow(isRunning);
if (isRunning && startedAt) {
return formatDurationSecs(Math.floor((Date.now() - startedAt) / 1000));
return formatDurationSecs(Math.floor((now - startedAt) / 1000));
}
return duration;
}
@ -605,16 +589,16 @@ export default function RunStages() {
);
const selectedStage = stages.find((s: Stage) => s.id === stageId) ?? stages[0];
const turnsQuery = useRunStageTurns(id, selectedStage?.id);
const hasStageTurns = (turnsQuery.data?.data.length ?? 0) > 0;
const shouldLoadEventFallback =
!!selectedStage?.id && !turnsQuery.isLoading && !turnsQuery.error && !hasStageTurns;
const eventsQuery = useRunEventsList(id, shouldLoadEventFallback);
const selectedStageId = selectedStage?.id;
const stageEventsQuery = useRunStageEvents(id, selectedStageId);
const turns = useMemo(
() => mapTurns(turnsQuery.data, eventsQuery.data, selectedStage?.id),
[eventsQuery.data, selectedStage?.id, turnsQuery.data],
() =>
selectedStageId
? eventsToActivity(stageEventsQuery.data ?? [], selectedStageId)
: [],
[stageEventsQuery.data, selectedStageId],
);
const isRunning = selectedStage?.status === "running";
const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false;
if (!id || !stages.length) {
return (
@ -636,11 +620,13 @@ export default function RunStages() {
<div className="min-w-0 flex-1 space-y-3">
<div className="sticky top-0 z-10 -mx-2 flex items-center gap-2 bg-page/85 px-2 py-2 backdrop-blur">
<SelectedIcon className={`size-5 ${selectedConfig.color} ${isRunning ? "animate-spin" : ""}`} />
<h3 className="text-base font-semibold text-fg">{selectedStage.name}</h3>
<SelectedIcon className={`size-5 ${selectedConfig.color} ${isActive ? "animate-spin" : ""}`} />
<h3 className="text-base font-semibold text-fg">
{formatStageLabel(selectedStage)}
</h3>
<span className="font-mono text-xs tabular-nums text-fg-muted">
<RunningStageDuration
isRunning={isRunning}
isRunning={isActive}
duration={selectedStage.duration}
/>
</span>

1
apps/fabro-web/app/routes/runs.test.tsx
View file

@ -25,6 +25,7 @@ describe("runs route board mapping", () => {
test("keeps blocked runs in the blocked lane and preserves question text", () => {
const columns = buildBoardColumns({
columns: [
{ id: "queued", name: "Queued" },
{ id: "initializing", name: "Initializing" },
{ id: "running", name: "Running" },
{ id: "blocked", name: "Blocked" },

24
apps/fabro-web/app/routes/runs.tsx
View file

@ -18,7 +18,7 @@ import {
arrayMove,
} from "@dnd-kit/sortable";
import { CSS } from "@dnd-kit/utilities";
import { ciConfig, columnStatusDisplay, deriveCiStatus, mapRunListItem } from "../data/runs";
import { ciConfig, columnStatusDisplay, columnStatuses, deriveCiStatus, mapRunListItem } from "../data/runs";
import type { CiStatus, CheckRun, CheckStatus, RunItem, RunWithStatus, ColumnStatus } from "../data/runs";
import { EmptyState } from "../components/state";
import { shouldRefreshBoardForEvent, useBoardEvents } from "../lib/board-events";
@ -38,6 +38,7 @@ interface ColumnStyle {
}
const columnStyles: Record<ColumnStatus, ColumnStyle> = {
queued: { iconType: "branch", actions: [] },
initializing: { iconType: "branch", actions: [] },
running: { iconType: "branch", actions: ["Watch", "Steer"] },
blocked: { iconType: "branch", actions: ["Answer Question"] },
@ -49,7 +50,7 @@ const defaultColumnStyle: ColumnStyle = { iconType: "branch", actions: [] };
const defaultColumnColors = { dot: "bg-fg-muted", text: "text-fg-muted" };
interface BoardRunsResponse {
columns: { id: string; name: string }[];
columns: PaginatedBoardRunList["columns"];
data: PaginatedBoardRunList["data"];
meta: PaginatedBoardRunList["meta"];
}
@ -64,16 +65,8 @@ type Column = {
items: RunItem[];
};
const SKELETON_STATUSES: ColumnStatus[] = [
"initializing",
"running",
"blocked",
"succeeded",
"failed",
];
function buildSkeletonColumns(): Column[] {
return SKELETON_STATUSES.map((id) => {
return columnStatuses.map((id) => {
const colors = columnStatusDisplay[id];
return {
id,
@ -98,7 +91,7 @@ export function buildBoardColumns(response: BoardRunsResponse): Column[] {
}
return response.columns.map((col) => {
const id = col.id as ColumnStatus;
const id = col.id;
const colors = columnStatusDisplay[id] ?? defaultColumnColors;
return {
id,
@ -754,6 +747,9 @@ export default function Runs() {
(sum, col) => sum + col.items.length,
0,
);
const visibleColumns = filteredColumns.filter(
(col) => col.id !== "queued" || col.items.length > 0,
);
return (
<DndContext sensors={sensors} collisionDetection={closestCenter} onDragEnd={handleDragEnd}>
@ -815,7 +811,7 @@ export default function Runs() {
{view === "columns" ? (
<>
<div className="flex gap-5 overflow-x-auto pb-4">
{filteredColumns.map((col) => (
{visibleColumns.map((col) => (
<div key={col.id} className="w-72 shrink-0">
<BoardColumn column={col} />
</div>
@ -838,7 +834,7 @@ export default function Runs() {
) : (
<>
<div className="space-y-4">
{filteredColumns.map((col) => {
{visibleColumns.map((col) => {
const isCollapsed = collapsed.has(col.id);
return (
<div key={col.id}>

189
docs/public/api-reference/fabro-api.yaml
View file

@ -1906,26 +1906,26 @@ paths:
schema:
$ref: "#/components/schemas/ErrorResponse"
/api/v1/runs/{id}/stages/{stageId}/turns:
/api/v1/runs/{id}/stages/{stageId}/events:
get:
operationId: listStageTurns
operationId: listStageEvents
tags: [Run Internals]
summary: List Stage Turns
description: Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations.
summary: List Stage Events
description: Returns a paginated JSON list of stored run events scoped to a single stage visit.
parameters:
- $ref: "#/components/parameters/RunId"
- $ref: "#/components/parameters/StageId"
- $ref: "#/components/parameters/PageLimit"
- $ref: "#/components/parameters/PageOffset"
- $ref: "#/components/parameters/SinceSeq"
- $ref: "#/components/parameters/EventLimit"
responses:
"200":
description: Paginated list of conversation turns
description: Paginated list of stage events
content:
application/json:
schema:
$ref: "#/components/schemas/PaginatedStageTurnList"
$ref: "#/components/schemas/PaginatedEventList"
"404":
description: Run or stage not found
description: Run not found.
headers:
x-request-id:
$ref: "#/components/headers/XRequestId"
@ -3856,20 +3856,6 @@ components:
meta:
$ref: "#/components/schemas/PaginationMeta"
PaginatedStageTurnList:
description: Paginated list of stage turns.
type: object
required:
- data
- meta
properties:
data:
type: array
items:
$ref: "#/components/schemas/StageTurn"
meta:
$ref: "#/components/schemas/PaginationMeta"
PaginatedApiQuestionList:
description: Paginated list of pending questions.
type: object
@ -5328,6 +5314,20 @@ components:
oneOf:
- $ref: "#/components/schemas/CommandTermination"
- type: "null"
started_at:
type: ["string", "null"]
format: date-time
description: Wall-clock time the latest attempt of this stage started, if known.
duration_ms:
type: ["integer", "null"]
format: uint64
minimum: 0
description: Wall-clock duration of the stage's latest terminal attempt, if known.
state:
oneOf:
- $ref: "#/components/schemas/StageState"
- type: "null"
description: Lifecycle state of the stage projection.
InterviewOption:
description: Option stored with an interview question in the event log.
@ -5656,6 +5656,7 @@ components:
description: Board column status for a run in the list view.
type: string
enum:
- queued
- initializing
- running
- blocked
@ -5669,7 +5670,7 @@ components:
- name
properties:
id:
type: string
$ref: "#/components/schemas/BoardColumn"
name:
type: string
@ -6159,7 +6160,7 @@ components:
example: 3501.0
BillingStageRef:
description: Reference to a billing stage.
description: Reference to a workflow node in a billing stage row.
type: object
required:
- id
@ -6319,11 +6320,13 @@ components:
- id
- name
- status
- node_id
- visit
properties:
id:
type: string
description: Unique stage identifier within the run.
example: propose-changes
description: StageId in "node_id@visit" form, e.g. verify@2.
example: verify@2
name:
type: string
description: Human-readable stage name.
@ -6334,107 +6337,21 @@ components:
type: number
description: Time spent in this stage, in seconds.
example: 154.0
dot_id:
node_id:
type: string
description: Node identifier in the Graphviz graph source.
example: propose
ToolUse:
description: A single tool invocation with its input, result, and execution metadata.
type: object
required:
- id
- tool_name
- input
- result
- is_error
properties:
id:
type: string
description: Unique identifier for this tool invocation. Enables correlation in parallel tool use.
example: toolu_01A09q90qw90lq917835lq9
tool_name:
type: string
description: Name of the tool that was invoked.
example: read_file
input:
type: string
description: JSON-encoded input passed to the tool.
example: '{ "path": "src/routes/auth.ts" }'
result:
type: string
description: Output returned by the tool. Contains the error message when is_error is true.
example: 'import { Router } from "express";'
is_error:
type: boolean
description: Whether the tool invocation failed. When true, the result field contains the error message.
example: false
duration_ms:
description: Node id in the workflow graph; multiple stages with different visits share the same node_id.
example: verify
visit:
type: integer
description: Wall-clock execution time of the tool invocation in milliseconds.
example: 142
StageTurn:
description: A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block.
discriminator:
propertyName: kind
mapping:
system: "#/components/schemas/SystemStageTurn"
assistant: "#/components/schemas/AssistantStageTurn"
tool: "#/components/schemas/ToolStageTurn"
oneOf:
- $ref: "#/components/schemas/SystemStageTurn"
- $ref: "#/components/schemas/AssistantStageTurn"
- $ref: "#/components/schemas/ToolStageTurn"
SystemStageTurn:
description: A system prompt turn that sets the stage's instructions.
type: object
required:
- kind
- content
properties:
kind:
type: string
enum: [system]
content:
type: string
description: System prompt text.
example: You are a drift detection agent. Compare the production and staging environments.
AssistantStageTurn:
description: An assistant response turn within a stage.
type: object
required:
- kind
- content
properties:
kind:
type: string
enum: [assistant]
content:
type: string
description: Assistant response text.
example: I'll start by loading the environment configurations for both production and staging.
ToolStageTurn:
description: A tool invocation turn containing one or more tool calls.
type: object
required:
- kind
- tools
properties:
kind:
type: string
enum: [tool]
content:
type: string
description: Text accompanying the tool invocations, or null when the turn contains only tool calls.
tools:
type: array
description: Tool invocations executed in this turn.
items:
$ref: "#/components/schemas/ToolUse"
format: uint32
minimum: 1
description: 1-based visit count; bumped each time the workflow re-enters this node.
example: 2
started_at:
type: ["string", "null"]
format: date-time
description: Wall-clock time the latest attempt of this stage started, if known.
example: "2026-04-29T12:34:56Z"
# ── File Diff Schemas ──────────────────────────────────────────────
@ -6607,7 +6524,7 @@ components:
# ── Billing Schemas ──────────────────────────────────────────────────
RunBillingStage:
description: Token counts and billed totals for a single stage within a run.
description: Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node.
type: object
required:
- stage
@ -6618,7 +6535,7 @@ components:
stage:
$ref: "#/components/schemas/BillingStageRef"
model:
description: Model used for this stage; null for non-LLM stages.
description: Latest usage-bearing visit model for this node; null when no visit used an LLM model.
oneOf:
- $ref: "#/components/schemas/ModelReference"
- type: "null"
@ -6626,8 +6543,18 @@ components:
$ref: "#/components/schemas/BilledTokenCounts"
runtime_secs:
type: number
description: Wall-clock runtime in seconds.
description: Wall-clock runtime in seconds, summed across every visit of this node.
example: 154.0
started_at:
type: ["string", "null"]
format: date-time
description: Wall-clock time the latest attempt of this stage started, if known.
example: "2026-04-29T12:34:56Z"
state:
oneOf:
- $ref: "#/components/schemas/StageState"
- type: "null"
description: Lifecycle state of the stage. Use to detect in-flight rows for client-side runtime ticking.
RunBillingTotals:
description: Aggregate billing totals across all stages of a run.
@ -6687,7 +6614,7 @@ components:
$ref: "#/components/schemas/ModelReference"
stages:
type: integer
description: Number of stages that used this model.
description: Number of usage-bearing stage visits that used this model.
example: 2
billing:
$ref: "#/components/schemas/BilledTokenCounts"
@ -6702,7 +6629,7 @@ components:
properties:
stages:
type: array
description: Per-stage billing breakdown.
description: Per-node billing breakdown. Each row sums billing and runtime across all visits of that node.
items:
$ref: "#/components/schemas/RunBillingStage"
totals:

106
docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md Normal file
View file

@ -0,0 +1,106 @@
# Cross-Tab SSE Coordination Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Coordinate Fabro web SSE subscriptions across tabs so one browser profile/origin opens at most one UI-owned `/api/v1/attach` EventSource in steady state when `BroadcastChannel` is available; brief overlap during election/takeover is tolerated and deduped.
**Architecture:** Add a browser-side SSE coordinator that elects one tab as leader, has that leader own the global EventSource, and broadcasts parsed run events to follower tabs over `BroadcastChannel`. Existing board and run-detail invalidation logic becomes a consumer of that global event feed, with the current per-tab SSE behavior preserved as a compatibility fallback.
**Tech Stack:** React, SWR, browser `BroadcastChannel`, browser `EventSource`, Bun tests, existing Fabro web API query keys.
---
## Summary
Build a browser-side SSE coordinator so Fabro web opens at most one `/api/v1/attach` EventSource per origin/browser profile in steady state when `BroadcastChannel` is available. Temporary duplicate leaders may exist during election/takeover, but event dedupe and generation checks make the overlap harmless and short-lived. The global stream becomes a shared cache-invalidation feed for both the runs board and run detail pages. No server API, OpenAPI, or Rust streaming contract changes are part of v1.
This supersedes the earlier web SSE limitation documented in `docs/plans/2026-04-19-002-feat-web-ui-lifecycle-actions-plan.md`: the old shared hook was code reuse only; this plan adds actual socket deduplication.
## Implementation Changes
- Add `apps/fabro-web/app/lib/cross-tab-sse.ts`.
- Export `subscribeToCrossTabSse(...)` with the same invalidation style as `subscribeToSharedEventSource`, a `resyncKeys` callback for gap recovery, and a `fallbackSubscribe` callback used when cross-tab coordination is unavailable.
- Use `BroadcastChannel` name `fabro:sse:v1`.
- Generate `tabId` with `crypto.randomUUID()` and a safe random fallback.
- Open one leader-owned `EventSource` to `queryKeys.system.attach()` (`/api/v1/attach`) in steady state.
- Leader dispatches each parsed `EventEnvelope` locally and broadcasts it to follower tabs.
- Followers do not open EventSource while a valid visible leader heartbeat exists; lower lexical `tabId` does not preempt a healthy visible leader.
- Implement leader election in the cross-tab module.
- Constants: `HEARTBEAT_MS = 1000`, `LEADER_STALE_MS = 4000`, `ELECTION_JITTER_MS = 150`.
- Messages: `hello`, `heartbeat`, `candidate`, `leader-changed`, `release`, `resync`, `event`.
- Define a typed message union. Every message includes `type`, `version: 1`, `tabId`, and `sentAt`.
- `heartbeat`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`.
- `candidate`: `{ type, version, tabId, sentAt, candidateId: tabId, candidateGeneration, visibility, observedLeaderId, observedGeneration, reason }`, where `reason` is `"hidden-leader" | "stale-leader" | "release" | "no-leader"`.
- `leader-changed`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`.
- `release`: `{ type, version, tabId, sentAt, leaderId, generation }`.
- `resync`: `{ type, version, tabId, sentAt, leaderId, generation, reason }`.
- `event`: `{ type, version, tabId, sentAt, leaderId, generation, payload }`.
- Use the candidacy phase for all leadership changes: hidden-leader takeover, stale-leader recovery, leader release, and no-leader startup.
- A candidate sets `candidateGeneration = observedGeneration + 1`, broadcasts `candidate`, waits jitter, and opens EventSource only if no higher-priority candidate for the same `candidateGeneration` appears.
- Candidate priority is election-scoped: visible candidates outrank hidden candidates; for equal visibility, lower lexical `candidateId` wins. This priority resolves elections and same-generation split brain only; it is not a reason to preempt a fresh visible leader.
- When a visible follower observes a fresh hidden leader heartbeat, it enters candidacy with `reason: "hidden-leader"`.
- When tabs detect a stale leader, leader release, or no known leader, they enter the same candidacy flow with the matching `reason`.
- If two visible candidates race for the same observed leader/generation, the lower lexical `candidateId` wins.
- Current leaders release when they observe a candidate whose `observedLeaderId` matches their `leaderId` and whose `observedGeneration` is current or newer.
- If same-generation split brain still occurs, lower-priority leaders release when they observe a same-generation higher-priority leader heartbeat or `leader-changed`.
- Each new leader uses `candidateGeneration`, broadcasts `leader-changed`, and followers ignore heartbeats/events from non-current leaders or stale generations.
- Hidden leader keeps the stream only when no visible candidate takes over.
- On `pagehide`/last local unsubscribe, a leader closes EventSource and broadcasts `release`.
- Brief split brain is tolerated; dedupe events by `payload.id`, falling back to `${run_id}:${seq}:${event}`.
- Keep dedupe bounded with a recent-event cache: max 1000 IDs and 5 minute TTL. Evict oldest entries when the max is exceeded and prune expired entries during event handling. Duplicate invalidations after eviction are acceptable; unbounded growth is not.
- Migrate consumers.
- `apps/fabro-web/app/lib/board-events.ts`: subscribe through the cross-tab global stream; keep existing board event allowlist.
- `apps/fabro-web/app/lib/run-events.ts`: subscribe through the same global stream, filter by `payload.run_id === runId`, and reuse `queryKeysForRunEvent`.
- In coordinated mode, run detail pages stay subscribed while mounted, including terminal runs, so post-terminal archive/unarchive changes can reconcile live.
- Do not close the global stream on `run.completed` / `run.failed`; terminal events only invalidate run-scoped keys.
- Keep `subscribeToSharedEventSource` in `apps/fabro-web/app/lib/sse.ts` for fallback and existing local sharing behavior.
- Gap and fallback behavior.
- If `BroadcastChannel` is unavailable or throws, call each subscriber's `fallbackSubscribe`.
- Board fallback uses the existing global `/api/v1/attach` path.
- Run detail fallback preserves the existing run-scoped `/api/v1/runs/:id/attach` path, so the old terminal-tab stale limitation remains only in fallback mode.
- Do not add replay to `/api/v1/attach`.
- On leader takeover, stale leader timeout, leader release, and new leader generation, broadcast `resync` or `leader-changed` so every tab with active local subscriptions runs its own `resyncKeys`.
- On `visibilitychange` back to visible without leadership change, run only that tab's local `resyncKeys`; do not broadcast cross-tab resync.
- Board `resyncKeys`: `queryKeys.boards.runs()`.
- Run `resyncKeys`: detail, files, billing, stages, events, LR graph, TB graph, and questions for that run.
## Tests
- Add `apps/fabro-web/app/lib/cross-tab-sse.test.ts` with fake `BroadcastChannel`, fake `EventSource`, and fake timers.
- One leader opens `/api/v1/attach`; followers open no EventSource.
- Leader broadcasts an event and all local subscribers receive invalidations.
- Run subscribers ignore events for other `run_id` values.
- Board and run subscriptions can coexist on the same global stream.
- Temporary duplicate leaders are allowed only during election/takeover and converge back to one leader.
- With fresh hidden-leader heartbeats, a visible tab broadcasts candidacy, hidden leader closes, visible tab opens `/api/v1/attach`, and followers resync.
- Two visible candidates racing for the same hidden leader resolve to the lower lexical `candidateId`.
- Two tabs detect the same stale leader simultaneously; only the winning candidate opens `/api/v1/attach` after jitter.
- Same-generation split brain converges to one leader by visibility, then lexical `tabId`.
- A fresh visible leader exists; a new visible follower with a lower lexical `tabId` joins and does not take leadership.
- A hidden leader does not release for a candidate that references an old `observedLeaderId` or stale `observedGeneration`.
- Stale heartbeat triggers takeover and every active tab calls its own `resyncKeys`.
- A follower tab calls its own board/run `resyncKeys` after another tab becomes leader.
- Duplicate event IDs are ignored.
- The recent-event dedupe cache evicts by TTL and max-size bound; duplicate invalidation may recur only after eviction.
- Stale heartbeat/event messages from an old leader/generation are ignored after takeover.
- Last unsubscribe closes leader EventSource and sends `release`.
- Missing/broken BroadcastChannel uses the per-tab fallback.
- Update existing tests.
- `apps/fabro-web/app/lib/board-events.test.tsx`: assert coordinated mode uses `/api/v1/attach` once and fallback preserves current behavior.
- `apps/fabro-web/app/lib/run-events.test.tsx`: assert coordinated mode filters by `run_id`; fallback preserves current `/runs/:id/attach` behavior; terminal run events do not close the global coordinator; terminal detail tabs still receive archive/unarchive invalidations while mounted in coordinated mode.
- Verification commands.
- `cd apps/fabro-web && bun test app/lib/cross-tab-sse.test.ts app/lib/board-events.test.tsx app/lib/run-events.test.tsx`
- `cd apps/fabro-web && bun run typecheck`
- Optional manual check: open 8 run-detail tabs for active runs; after election settles and with BroadcastChannel available, Chrome should show one active `/api/v1/attach` EventStream across the participating tabs and no UI-created `/api/v1/runs/:id/attach` streams.
## Assumptions
- Treat SSE as live cache invalidation, not an exact gapless event log.
- Preserve current run-specific attach endpoint for CLI, API clients, and fallback only.
- Do not touch unrelated dirty worktree files; intentionally replace or adapt the earlier interrupted SSE test edits as part of the new test suite.
- No docs or public API updates are required for v1 because this is an internal web transport change.

57
lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs
View file

@ -1,4 +1,5 @@
use fabro_api::types::RunBillingStage;
use fabro_types::StageState;
use serde_json::json;
#[test]
@ -28,3 +29,59 @@ fn run_billing_stage_model_accepts_required_null() {
assert!(encoded.get("model").is_some());
assert!(encoded["model"].is_null());
}
#[test]
fn run_billing_stage_round_trips_terminal_row_with_started_at_and_state() {
let value = json!({
"stage": {
"id": "build",
"name": "build"
},
"model": { "id": "claude-sonnet-4-5" },
"billing": {
"input_tokens": 12,
"output_tokens": 34,
"total_tokens": 46,
"reasoning_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0
},
"runtime_secs": 5.5,
"started_at": "2026-04-29T12:34:56Z",
"state": "succeeded"
});
let stage: RunBillingStage =
serde_json::from_value(value.clone()).expect("terminal stage row should deserialize");
assert!(stage.started_at.is_some());
assert_eq!(stage.state, Some(StageState::Succeeded));
assert_eq!(serde_json::to_value(stage).unwrap(), value);
}
#[test]
fn run_billing_stage_round_trips_in_flight_row() {
let value = json!({
"stage": {
"id": "build",
"name": "build"
},
"model": null,
"billing": {
"input_tokens": 0,
"output_tokens": 0,
"total_tokens": 0,
"reasoning_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0
},
"runtime_secs": 1.25,
"started_at": "2026-04-29T12:34:56Z",
"state": "running"
});
let stage: RunBillingStage =
serde_json::from_value(value.clone()).expect("in-flight stage row should deserialize");
assert!(stage.model.is_none());
assert_eq!(stage.state, Some(StageState::Running));
assert_eq!(serde_json::to_value(stage).unwrap(), value);
}

5
lib/crates/fabro-api/tests/stage_projection_round_trip.rs
View file

@ -28,7 +28,10 @@ fn stage_projection_round_trips_representative_json() {
"parallel_results": [{ "branch": 0, "status": "succeeded" }],
"stdout": "ok",
"stderr": "",
"termination": "exited"
"termination": "exited",
"started_at": "2026-04-29T12:34:00Z",
"duration_ms": 56000,
"state": "succeeded"
});
let state: StageProjection = serde_json::from_value(value.clone()).unwrap();

12
lib/crates/fabro-cli/src/commands/auth/status.rs
View file

@ -1,5 +1,5 @@
use anyhow::Result;
use chrono::{DateTime, Utc};
use chrono::{DateTime, SecondsFormat, Utc};
use fabro_client::{AuthEntry, AuthStore, OAuthEntry};
use serde::Serialize;
@ -108,12 +108,12 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res
fabro_util::printerr!(
printer,
" Access expires: {}",
access_token_expires_at.to_rfc3339()
access_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true)
);
fabro_util::printerr!(
printer,
" Refresh expires: {}",
refresh_token_expires_at.to_rfc3339()
refresh_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true)
);
}
StatusRow::DevToken {
@ -122,7 +122,11 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res
} => {
fabro_util::printerr!(printer, "{server}");
fabro_util::printerr!(printer, " Auth: dev-token");
fabro_util::printerr!(printer, " Logged in: {}", logged_in_at.to_rfc3339());
fabro_util::printerr!(
printer,
" Logged in: {}",
logged_in_at.to_rfc3339_opts(SecondsFormat::Secs, true)
);
}
}
}

51
lib/crates/fabro-cli/src/commands/run/logs.rs
View file

@ -14,7 +14,7 @@ use std::time::Duration;
use anyhow::{Context, Result, bail};
use chrono::{DateTime, Utc};
use fabro_redact::redact_jsonl_line;
use fabro_types::run_event::is_metadata_snapshot_compat_notice_code;
use fabro_types::RunNoticeCode;
use fabro_util::json::normalize_json_value;
use fabro_util::terminal::Styles;
use tokio::time;
@ -801,7 +801,9 @@ fn format_event_pretty_value(envelope: &serde_json::Value, styles: &Styles) -> O
}
fn is_metadata_snapshot_compat_notice(envelope: &serde_json::Value) -> bool {
prop_str_field(envelope, "code").is_some_and(is_metadata_snapshot_compat_notice_code)
prop_str_field(envelope, "code")
.and_then(|code| code.parse::<RunNoticeCode>().ok())
.is_some_and(RunNoticeCode::is_metadata_snapshot_compat)
}
fn str_field<'a>(value: &'a serde_json::Value, key: &str) -> Option<&'a str> {
@ -1150,14 +1152,27 @@ mod tests {
#[test]
fn pretty_run_notice_warn() {
let styles = no_color_styles();
let line = r#"{"ts":"2026-01-01T14:25:00Z","event":"run.notice","properties":{"level":"warn","code":"sandbox_cleanup_failed","message":"sandbox cleanup failed: boom"}}"#;
let result = format_event_pretty(line, &styles).unwrap();
let code = RunNoticeCode::SandboxCleanupFailed.to_string();
let line = serde_json::json!({
"ts": "2026-01-01T14:25:00Z",
"event": "run.notice",
"properties": {
"level": "warn",
"code": code,
"message": "sandbox cleanup failed: boom",
},
})
.to_string();
let result = format_event_pretty(&line, &styles).unwrap();
assert!(result.contains("Warning:"), "got: {result}");
assert!(
result.contains("sandbox cleanup failed: boom"),
"got: {result}"
);
assert!(result.contains("[sandbox_cleanup_failed]"), "got: {result}");
assert!(
result.contains(&format!("[{}]", RunNoticeCode::SandboxCleanupFailed)),
"got: {result}"
);
}
#[test]
@ -1235,13 +1250,31 @@ mod tests {
fn pretty_stream_suppresses_metadata_compat_notice_only() {
let styles = no_color_styles();
let failed = r#"{"ts":"2026-01-01T14:25:00Z","event":"metadata.snapshot.failed","properties":{"phase":"checkpoint","branch":"fabro/meta","duration_ms":900,"failure_kind":"write","error":"write failed"}}"#;
let compat_notice = r#"{"ts":"2026-01-01T14:25:01Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_write_failed","message":"legacy metadata warning"}}"#;
let degraded_notice = r#"{"ts":"2026-01-01T14:25:02Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_degraded","message":"metadata snapshots disabled"}}"#;
let compat_notice = serde_json::json!({
"ts": "2026-01-01T14:25:01Z",
"event": "run.notice",
"properties": {
"level": "warn",
"code": RunNoticeCode::CheckpointMetadataWriteFailed,
"message": "legacy metadata warning",
},
})
.to_string();
let degraded_notice = serde_json::json!({
"ts": "2026-01-01T14:25:02Z",
"event": "run.notice",
"properties": {
"level": "warn",
"code": RunNoticeCode::CheckpointMetadataDegraded,
"message": "metadata snapshots disabled",
},
})
.to_string();
let mut state = PrettyEventState::default();
assert!(format_event_pretty_streamed(failed, &styles, &mut state).is_some());
assert!(format_event_pretty_streamed(compat_notice, &styles, &mut state).is_none());
let degraded = format_event_pretty_streamed(degraded_notice, &styles, &mut state).unwrap();
assert!(format_event_pretty_streamed(&compat_notice, &styles, &mut state).is_none());
let degraded = format_event_pretty_streamed(&degraded_notice, &styles, &mut state).unwrap();
assert!(
degraded.contains("metadata snapshots disabled"),
"got: {degraded}"

7
lib/crates/fabro-cli/src/commands/run/run_progress/event.rs
View file

@ -527,7 +527,7 @@ fn display_value(value: &Value) -> Option<String> {
mod tests {
use fabro_agent::AgentEvent;
use fabro_types::{MetadataSnapshotFailureKind, MetadataSnapshotPhase, fixtures};
use fabro_workflow::event::{Event, to_run_event};
use fabro_workflow::event::{Event, RunNoticeCode, to_run_event};
use super::*;
@ -804,10 +804,11 @@ mod tests {
fn round_trip_run_notice() {
let event = Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "sandbox_cleanup_failed".into(),
code: RunNoticeCode::SandboxCleanupFailed.to_string(),
message: "sandbox cleanup failed".into(),
exec_output_tail: None,
};
let expected_code = RunNoticeCode::SandboxCleanupFailed.to_string();
let stored = to_run_event(&fixtures::RUN_1, &event);
let parsed = from_run_event(&stored).unwrap();
@ -817,7 +818,7 @@ mod tests {
level: RunNoticeLevel::Warn,
code,
message,
} if code == "sandbox_cleanup_failed" && message == "sandbox cleanup failed"
} if code == expected_code && message == "sandbox cleanup failed"
));
}

14
lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs
View file

@ -3,8 +3,7 @@
reason = "sync CLI run-progress renderer: writes to std::io::stderr directly"
)]
use fabro_types::RunEvent;
use fabro_types::run_event::is_metadata_snapshot_compat_notice_code;
use fabro_types::{RunEvent, RunNoticeCode};
mod event;
mod info_display;
@ -444,7 +443,10 @@ impl ProgressUI {
message,
} => {
if self.saw_metadata_snapshot_failure
&& is_metadata_snapshot_compat_notice_code(&code)
&& code
.parse::<RunNoticeCode>()
.ok()
.is_some_and(RunNoticeCode::is_metadata_snapshot_compat)
{
return;
}
@ -1208,7 +1210,7 @@ mod tests {
emit(&mut ui, Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "sandbox_cleanup_failed".into(),
code: RunNoticeCode::SandboxCleanupFailed.to_string(),
message: "sandbox cleanup failed".into(),
exec_output_tail: None,
});
@ -1279,13 +1281,13 @@ mod tests {
});
emit(&mut ui, Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "checkpoint_metadata_write_failed".into(),
code: RunNoticeCode::CheckpointMetadataWriteFailed.to_string(),
message: "legacy metadata warning".into(),
exec_output_tail: None,
});
emit(&mut ui, Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "checkpoint_metadata_degraded".into(),
code: RunNoticeCode::CheckpointMetadataDegraded.to_string(),
message: "metadata snapshots are disabled for this run".into(),
exec_output_tail: None,
});

17
lib/crates/fabro-cli/tests/it/cmd/attach.rs
View file

@ -160,6 +160,7 @@ fn attach_replays_completed_detached_run() {
----- stdout -----
----- stderr -----
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Run Tests [TIME]
@ -267,6 +268,7 @@ fn attach_before_completion_streams_to_finished_state() {
----- stdout -----
----- stderr -----
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [DURATION]
✓ wait [DURATION]
@ -699,6 +701,21 @@ fn attach_json_errors_without_prompting_for_human_input() {
"run_id": "[ULID]",
"ts": "[TIMESTAMP]"
},
{
"actor": {
"kind": "worker",
"run_id": "[ULID]"
},
"event": "run.notice",
"id": "[EVENT_ID]",
"properties": {
"code": "worktree_skipped_no_git",
"level": "warn",
"message": "Worktree mode `always` requested but no Git repository was found; running without a worktree."
},
"run_id": "[ULID]",
"ts": "[TIMESTAMP]"
},
{
"actor": {
"kind": "worker",

6
lib/crates/fabro-cli/tests/it/cmd/dump.rs
View file

@ -278,9 +278,9 @@ fn dump_exports_completed_run_snapshot() {
");
assert_snapshot!(dump_file_summary(&output_dir), @"
checkpoints/0013.json
checkpoints/0017.json
checkpoints/0021.json
checkpoints/0014.json
checkpoints/0018.json
checkpoints/0022.json
events.jsonl
graph.fabro
run.json

1
lib/crates/fabro-cli/tests/it/cmd/run.rs
View file

@ -684,6 +684,7 @@ fn dry_run_simple() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Run Tests [TIME]

5
lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs
View file

@ -21,6 +21,7 @@ fn dry_run_branching() {
warning [node: implement]: Node 'implement' has goal_gate=true but no retry_target or fallback_retry_target (goal_gate_has_retry)
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Plan [TIME]
@ -57,6 +58,7 @@ fn dry_run_conditions() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [TIME]
✓ Decide [TIME]
@ -91,6 +93,7 @@ fn dry_run_parallel() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [TIME]
✓ Fork Work [TIME]
@ -126,6 +129,7 @@ fn dry_run_styled() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [TIME]
✓ Plan [TIME]
@ -161,6 +165,7 @@ fn dry_run_legacy_tool() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Echo [TIME]

7
lib/crates/fabro-dump/src/lib.rs
View file

@ -66,18 +66,13 @@ impl RunDump {
entries.push(RunDumpEntry::text("graph.fabro", graph_source.clone()));
}
let mut stages: Vec<_> = state.iter_stages().collect();
let stages: Vec<_> = state.iter_stages().collect();
if stages.len() > MAX_STAGES_IN_DUMP {
bail!(
"run dump supports at most {MAX_STAGES_IN_DUMP} stages with the current path prefix width (got {})",
stages.len()
);
}
stages.sort_by(|(left_id, left), (right_id, right)| {
left.first_event_seq
.cmp(&right.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
let mut stage_ranks = HashMap::new();
for (index, (stage_id, _)) in stages.iter().enumerate() {

59
lib/crates/fabro-sandbox/src/local.rs
View file

@ -127,6 +127,19 @@ fn process_env_vars() -> Vec<(String, String)> {
std::env::vars().collect()
}
async fn drain_pipe<R>(mut pipe: Option<R>, stream: CommandOutputStream) -> String
where
R: AsyncRead + Unpin,
{
let mut buf = String::new();
if let Some(ref mut reader) = pipe {
if let Err(err) = reader.read_to_string(&mut buf).await {
tracing::warn!(error = %err, ?stream, "Failed to drain child output");
}
}
buf
}
#[async_trait]
impl Sandbox for LocalSandbox {
async fn read_file(
@ -278,22 +291,12 @@ impl Sandbox for LocalSandbox {
// it writes more than the OS pipe buffer (~64 KB) the write() syscall
// blocks until the parent drains the pipe, but the parent is blocked
// on child.wait().
let mut stdout_pipe = child.stdout.take();
let mut stderr_pipe = child.stderr.take();
let stdout_task = tokio::spawn(async move {
let mut buf = String::new();
if let Some(ref mut r) = stdout_pipe {
let _ = r.read_to_string(&mut buf).await;
}
buf
});
let stderr_task = tokio::spawn(async move {
let mut buf = String::new();
if let Some(ref mut r) = stderr_pipe {
let _ = r.read_to_string(&mut buf).await;
}
buf
});
let stdout_pipe = child.stdout.take();
let stderr_pipe = child.stderr.take();
let stdout_task =
tokio::spawn(async move { drain_pipe(stdout_pipe, CommandOutputStream::Stdout).await });
let stderr_task =
tokio::spawn(async move { drain_pipe(stderr_pipe, CommandOutputStream::Stderr).await });
let (termination, exit_code) = tokio::select! {
status_result = child.wait() => {
@ -714,7 +717,12 @@ where
)]
mod tests {
use std::collections::HashMap;
use std::io;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context as TaskContext, Poll};
use tokio::io::ReadBuf;
use super::*;
@ -724,6 +732,25 @@ mod tests {
dir
}
#[tokio::test]
async fn drain_pipe_returns_empty_buffer_after_read_failure() {
struct FailingReader;
impl AsyncRead for FailingReader {
fn poll_read(
self: Pin<&mut Self>,
_cx: &mut TaskContext<'_>,
_buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
Poll::Ready(Err(io::Error::other("simulated read failure")))
}
}
let output = drain_pipe(Some(FailingReader), CommandOutputStream::Stdout).await;
assert!(output.is_empty());
}
#[tokio::test]
async fn read_file_with_line_numbers() {
let dir = temp_dir();

273
lib/crates/fabro-server/src/demo/mod.rs
View file

@ -15,15 +15,16 @@ use axum::http::StatusCode;
use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response};
use fabro_api::types::{
CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, FileDiff, FileDiffChangeKind,
PaginatedRunFileList, RunArtifactListResponse, RunFilesMeta,
CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, EventEnvelope, FileDiff,
FileDiffChangeKind, PaginatedEventList, PaginatedRunFileList, PaginationMeta,
RunArtifactListResponse, RunFilesMeta,
};
use serde_json::json;
use crate::error::ApiError;
use crate::principal_middleware::RequiredUser;
use crate::run_selector::{ResolveRunError, resolve_run_by_selector};
use crate::server::{AppState, PaginationParams};
use crate::server::{AppState, EventListParams, PaginationParams, parse_stage_id_path};
fn paginated_response<T: serde::Serialize>(
items: Vec<T>,
@ -133,13 +134,39 @@ pub(crate) async fn get_run_stages(
paginated_response(runs::stages(), &pagination)
}
pub(crate) async fn get_stage_turns(
pub(crate) async fn get_stage_events(
_auth: RequiredUser,
State(_state): State<Arc<AppState>>,
Path((_id, _stage_id)): Path<(String, String)>,
Query(pagination): Query<PaginationParams>,
Path((_id, stage_id)): Path<(String, String)>,
Query(params): Query<EventListParams>,
) -> Response {
paginated_response(runs::turns(), &pagination)
let stage_id = match parse_stage_id_path(&stage_id) {
Ok(stage_id) => stage_id,
Err(response) => return response,
};
let since_seq = params.since_seq();
let limit = params.limit();
let mut matches: Vec<EventEnvelope> = runs::stage_events()
.into_iter()
.filter(|envelope| {
envelope.seq >= since_seq
&& (envelope.event.stage_id.as_ref() == Some(&stage_id)
|| (envelope.event.stage_id.is_none()
&& stage_id.visit() == 1
&& envelope.event.node_id.as_deref() == Some(stage_id.node_id())))
})
.take(limit + 1)
.collect();
let has_more = matches.len() > limit;
matches.truncate(limit);
(
StatusCode::OK,
Json(PaginatedEventList {
data: matches,
meta: PaginationMeta { has_more },
}),
)
.into_response()
}
pub(crate) async fn list_run_artifacts_stub(
@ -784,9 +811,10 @@ mod runs {
RunNamespace, RunPrepareSettings, RunSandboxSettings,
};
use fabro_types::settings::{InterpString, ProjectNamespace, WorkflowNamespace};
use fabro_types::{RunId, WorkflowSettings};
use fabro_types::{RunId, StageId, WorkflowSettings};
use super::ts;
use crate::server::run_stage_from_stage_id;
fn labels(entries: &[(&str, &str)]) -> HashMap<String, String> {
entries
@ -795,8 +823,8 @@ mod runs {
.collect()
}
fn demo_run_ids() -> &'static [RunId; 6] {
static IDS: OnceLock<[RunId; 6]> = OnceLock::new();
fn demo_run_ids() -> &'static [RunId; 7] {
static IDS: OnceLock<[RunId; 7]> = OnceLock::new();
IDS.get_or_init(|| {
[
RunId::with_timestamp(ts("2026-03-06T14:30:00Z"), 1),
@ -805,6 +833,7 @@ mod runs {
RunId::with_timestamp(ts("2026-03-04T10:00:00Z"), 4),
RunId::with_timestamp(ts("2026-03-03T16:45:00Z"), 5),
RunId::with_timestamp(ts("2026-02-28T14:00:00Z"), 6),
RunId::with_timestamp(ts("2026-03-06T14:35:00Z"), 7),
]
})
}
@ -974,23 +1003,27 @@ mod runs {
pub(super) fn columns() -> Vec<BoardColumnDefinition> {
vec![
BoardColumnDefinition {
id: "initializing".into(),
id: BoardColumn::Queued,
name: "Queued".into(),
},
BoardColumnDefinition {
id: BoardColumn::Initializing,
name: "Initializing".into(),
},
BoardColumnDefinition {
id: "running".into(),
id: BoardColumn::Running,
name: "Running".into(),
},
BoardColumnDefinition {
id: "blocked".into(),
id: BoardColumn::Blocked,
name: "Blocked".into(),
},
BoardColumnDefinition {
id: "succeeded".into(),
id: BoardColumn::Succeeded,
name: "Succeeded".into(),
},
BoardColumnDefinition {
id: "failed".into(),
id: BoardColumn::Failed,
name: "Failed".into(),
},
]
@ -1082,6 +1115,20 @@ mod runs {
Some(720000),
&[("release", "preview")],
),
summary(
7,
"api-server",
"implement",
"Implement",
"Add audit log retention policy",
"queued",
"2026-03-06T14:35:00Z",
None,
None,
None,
None,
&[("owner", "platform")],
),
]
}
@ -1150,54 +1197,164 @@ mod runs {
None,
None,
),
board_item(
take_summary(&mut summaries, demo_run_id(7)),
BoardColumn::Queued,
None,
None,
None,
),
]
}
pub(super) fn stages() -> Vec<RunStage> {
vec![
RunStage {
id: "detect-drift".into(),
name: "Detect Drift".into(),
status: StageState::Succeeded,
duration_secs: Some(72.0),
dot_id: Some("detect".into()),
},
RunStage {
id: "propose-changes".into(),
name: "Propose Changes".into(),
status: StageState::Succeeded,
duration_secs: Some(154.0),
dot_id: Some("propose".into()),
},
RunStage {
id: "review-changes".into(),
name: "Review Changes".into(),
status: StageState::Succeeded,
duration_secs: Some(45.0),
dot_id: Some("review".into()),
},
RunStage {
id: "apply-changes".into(),
name: "Apply Changes".into(),
status: StageState::Running,
duration_secs: Some(118.0),
dot_id: Some("apply".into()),
},
run_stage_from_stage_id(
&StageId::new("detect-drift", 1),
"Detect Drift",
StageState::Succeeded,
Some(72.0),
None,
),
run_stage_from_stage_id(
&StageId::new("propose-changes", 1),
"Propose Changes",
StageState::Succeeded,
Some(154.0),
None,
),
run_stage_from_stage_id(
&StageId::new("review-changes", 1),
"Review Changes",
StageState::Succeeded,
Some(45.0),
None,
),
run_stage_from_stage_id(
&StageId::new("apply-changes", 1),
"Apply Changes",
StageState::Succeeded,
Some(118.0),
None,
),
run_stage_from_stage_id(
&StageId::new("apply-changes", 2),
"Apply Changes",
StageState::Running,
None,
None,
),
]
}
pub(super) fn turns() -> Vec<StageTurn> {
pub(super) fn stage_events() -> Vec<fabro_types::EventEnvelope> {
use fabro_model::BilledTokenCounts;
use fabro_types::run_event::agent::{
AgentMessageProps, AgentToolCompletedProps, AgentToolStartedProps,
};
use fabro_types::run_event::stage::StagePromptProps;
use fabro_types::{EventBody, EventEnvelope, RunEvent};
let run_id = demo_run_id(1);
let node_id = "detect-drift";
let stage_id = fabro_types::StageId::new(node_id, 1);
let ts = ts("2026-03-06T14:30:00Z");
let make_envelope = |seq: u32, id: &str, body: EventBody| EventEnvelope {
seq,
event: RunEvent {
id: id.into(),
ts,
run_id,
node_id: Some(node_id.into()),
node_label: Some("Detect Drift".into()),
stage_id: Some(stage_id.clone()),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body,
},
};
vec![
StageTurn::SystemStageTurn(SystemStageTurn { kind: SystemStageTurnKind::System, content: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into() }),
StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I'll start by loading the environment configurations for both production and staging to compare them.".into() }),
StageTurn::ToolStageTurn(ToolStageTurn {
kind: ToolStageTurnKind::Tool, content: None,
tools: vec![
ToolUse { id: "toolu_01".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/production/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-prod.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(45) },
ToolUse { id: "toolu_02".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/staging/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-staging.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(38) },
],
}),
StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into() }),
make_envelope(
1,
"evt-detect-drift-1",
EventBody::StagePrompt(StagePromptProps {
visit: 1,
text: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into(),
mode: None,
provider: None,
model: None,
}),
),
make_envelope(
2,
"evt-detect-drift-2",
EventBody::AgentMessage(AgentMessageProps {
text: "I'll start by loading the environment configurations for both production and staging to compare them.".into(),
model: "Opus 4.6".into(),
billing: BilledTokenCounts::default(),
tool_call_count: 0,
visit: 1,
}),
),
make_envelope(
3,
"evt-detect-drift-3",
EventBody::AgentToolStarted(AgentToolStartedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_01".into(),
arguments: serde_json::json!({ "path": "environments/production/config.toml" }),
visit: 1,
}),
),
make_envelope(
4,
"evt-detect-drift-4",
EventBody::AgentToolCompleted(AgentToolCompletedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_01".into(),
output: serde_json::json!("[redis]\nhost = \"redis-prod.internal\"\nport = 6379"),
is_error: false,
visit: 1,
}),
),
make_envelope(
5,
"evt-detect-drift-5",
EventBody::AgentToolStarted(AgentToolStartedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_02".into(),
arguments: serde_json::json!({ "path": "environments/staging/config.toml" }),
visit: 1,
}),
),
make_envelope(
6,
"evt-detect-drift-6",
EventBody::AgentToolCompleted(AgentToolCompletedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_02".into(),
output: serde_json::json!("[redis]\nhost = \"redis-staging.internal\"\nport = 6379"),
is_error: false,
visit: 1,
}),
),
make_envelope(
7,
"evt-detect-drift-7",
EventBody::AgentMessage(AgentMessageProps {
text: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into(),
model: "Opus 4.6".into(),
billing: BilledTokenCounts::default(),
tool_call_count: 0,
visit: 1,
}),
),
]
}
@ -1222,6 +1379,8 @@ mod runs {
total_usd_micros: Some(480_000),
},
runtime_secs: 72.0,
started_at: None,
state: Some(StageState::Succeeded),
},
RunBillingStage {
stage: BillingStageRef {
@ -1241,6 +1400,8 @@ mod runs {
total_usd_micros: Some(720_000),
},
runtime_secs: 154.0,
started_at: None,
state: Some(StageState::Succeeded),
},
RunBillingStage {
stage: BillingStageRef {
@ -1260,6 +1421,8 @@ mod runs {
total_usd_micros: Some(190_000),
},
runtime_secs: 45.0,
started_at: None,
state: Some(StageState::Succeeded),
},
RunBillingStage {
stage: BillingStageRef {
@ -1279,6 +1442,8 @@ mod runs {
total_usd_micros: Some(870_000),
},
runtime_secs: 118.0,
started_at: None,
state: Some(StageState::Running),
},
],
totals: RunBillingTotals {

18
lib/crates/fabro-server/src/principal_middleware.rs
View file

@ -52,6 +52,7 @@ pub(crate) struct RequestAuth(pub(crate) AuthContextSlot);
pub(crate) struct RequiredUser(pub(crate) UserPrincipal);
pub(crate) struct RequireRunScoped(pub(crate) RunId);
pub(crate) struct RequireRunBlob(pub(crate) RunId, pub(crate) RunBlobId);
pub(crate) struct RequireRunStageScoped(pub(crate) RunId, pub(crate) String);
pub(crate) struct RequireStageArtifact(pub(crate) RunId, pub(crate) StageId);
pub(crate) struct RequireCommandLog(
pub(crate) RunId,
@ -203,6 +204,23 @@ impl FromRequestParts<Arc<AppState>> for RequireRunBlob {
}
}
impl FromRequestParts<Arc<AppState>> for RequireRunStageScoped {
type Rejection = Response;
async fn from_request_parts(
parts: &mut Parts,
state: &Arc<AppState>,
) -> Result<Self, Self::Rejection> {
let Path((id, stage_id)): Path<(String, String)> = Path::from_request_parts(parts, state)
.await
.map_err(IntoResponse::into_response)?;
let run_id = parse_run_id_path(&id)?;
require_worker_or_user_for_run(&auth_slot_from_parts(parts), &run_id)
.map_err(IntoResponse::into_response)?;
Ok(Self(run_id, stage_id))
}
}
impl FromRequestParts<Arc<AppState>> for RequireStageArtifact {
type Rejection = Response;

132
lib/crates/fabro-server/src/server.rs
View file

@ -54,7 +54,7 @@ use fabro_llm::types::{
ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Role, ToolChoice,
ToolDefinition,
};
use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog, ModelTestMode, Provider};
use fabro_model::{BilledTokenCounts, Catalog, ModelTestMode, Provider};
use fabro_redact::redact_jsonl_line;
use fabro_sandbox::daytona::{self, DaytonaSandbox};
use fabro_sandbox::reconnect::reconnect;
@ -126,7 +126,7 @@ use crate::ip_allowlist::{IpAllowlistConfig, ip_allowlist_middleware};
use crate::jwt_auth::{self, AuthMode};
use crate::principal_middleware::{
AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunScoped,
RequireStageArtifact, RequiredUser, principal_middleware,
RequireRunStageScoped, RequireStageArtifact, RequiredUser, principal_middleware,
};
use crate::request_id::{self, RequestId};
use crate::run_files::{FilesInFlight, new_files_in_flight};
@ -139,6 +139,7 @@ use crate::{
mod handler;
pub(crate) use handler::events::EventListParams;
#[cfg(test)]
pub(in crate::server) use handler::events::filtered_global_events;
pub(crate) use handler::graph::render_graph_bytes;
@ -537,17 +538,50 @@ pub(crate) struct ResolvedAppStateSettings {
pub(crate) manifest_run_settings: std::result::Result<RunNamespace, SharedError>,
}
fn accumulate_model_billing(entry: &mut ModelBillingTotals, usage: &BilledModelUsage) {
let tokens = usage.tokens();
entry.stages += 1;
entry.billing.input_tokens += tokens.input_tokens;
entry.billing.output_tokens += tokens.output_tokens;
entry.billing.reasoning_tokens += tokens.reasoning_tokens;
entry.billing.cache_read_tokens += tokens.cache_read_tokens;
entry.billing.cache_write_tokens += tokens.cache_write_tokens;
entry.billing.total_tokens += tokens.total_tokens();
if let Some(value) = usage.total_usd_micros {
*entry.billing.total_usd_micros.get_or_insert(0) += value;
fn accumulate_billed_token_counts(target: &mut BilledTokenCounts, source: &BilledTokenCounts) {
target.input_tokens += source.input_tokens;
target.output_tokens += source.output_tokens;
target.reasoning_tokens += source.reasoning_tokens;
target.cache_read_tokens += source.cache_read_tokens;
target.cache_write_tokens += source.cache_write_tokens;
target.total_tokens += source.total_tokens;
if let Some(value) = source.total_usd_micros {
*target.total_usd_micros.get_or_insert(0) += value;
}
}
fn accumulate_billing_rollup(
accumulator: &mut BillingAccumulator,
rollup: &fabro_workflow::ProjectionBillingRollup,
) {
accumulator.total_runs += 1;
accumulator.total_runtime_secs += rollup.runtime_ms as f64 / 1000.0;
for model in &rollup.by_model {
let entry = accumulator
.by_model
.entry(model.model_id.clone())
.or_default();
entry.stages += model.stages;
accumulate_billed_token_counts(&mut entry.billing, &model.billing);
}
}
pub(crate) fn run_stage_from_stage_id(
stage_id: &StageId,
name: impl Into<String>,
status: StageState,
duration_secs: Option<f64>,
started_at: Option<chrono::DateTime<chrono::Utc>>,
) -> RunStage {
RunStage {
id: stage_id.to_string(),
name: name.into(),
status,
duration_secs,
node_id: stage_id.node_id().to_string(),
visit: std::num::NonZeroU32::new(stage_id.visit())
.expect("StageId stores a non-zero visit"),
started_at,
}
}
@ -2777,9 +2811,9 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
}
}
// Save final checkpoint
let checkpoint = match run_store.state().await {
Ok(state) => state.checkpoint,
// Save final projection
let final_projection = match run_store.state().await {
Ok(state) => Some(state),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state from store");
None
@ -2787,32 +2821,17 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
// Accumulate aggregate usage after execution completes.
if let Some(ref cp) = checkpoint {
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store");
HashMap::default()
}
};
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
agg.total_runs += 1;
let mut run_runtime: f64 = 0.0;
for (node_id, outcome) in &cp.node_outcomes {
if let Some(usage) = &outcome.usage {
let entry = agg
.by_model
.entry(usage.model_id().to_string())
.or_default();
accumulate_model_billing(entry, usage);
}
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
run_runtime += duration_ms as f64 / 1000.0;
if let Some(ref projection) = final_projection {
if projection.checkpoint.is_some() {
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
accumulate_billing_rollup(
&mut agg,
&fabro_workflow::billing_rollup_from_projection(projection),
);
}
agg.total_runtime_secs += run_runtime;
}
let mut runs = state.runs.lock().expect("runs lock poisoned");
@ -2861,7 +2880,9 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
}
}
managed_run.checkpoint = checkpoint;
managed_run.checkpoint = final_projection
.as_ref()
.and_then(|projection| projection.checkpoint.clone());
managed_run.run_dir = Some(run_dir);
clear_live_run_state(managed_run);
}
@ -3104,32 +3125,15 @@ async fn execute_run_subprocess(state: Arc<AppState>, run_id: RunId) {
}
};
if let Some(ref checkpoint) = final_state.checkpoint {
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store");
HashMap::default()
}
};
if final_state.checkpoint.is_some() {
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
agg.total_runs += 1;
let mut run_runtime: f64 = 0.0;
for (node_id, outcome) in &checkpoint.node_outcomes {
if let Some(usage) = &outcome.usage {
let entry = agg
.by_model
.entry(usage.model_id().to_string())
.or_default();
accumulate_model_billing(entry, usage);
}
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
run_runtime += duration_ms as f64 / 1000.0;
}
agg.total_runtime_secs += run_runtime;
accumulate_billing_rollup(
&mut agg,
&fabro_workflow::billing_rollup_from_projection(&final_state),
);
}
let mut runs = state.runs.lock().expect("runs lock poisoned");

317
lib/crates/fabro-server/src/server/handler/billing.rs
View file

@ -1,13 +1,14 @@
use std::collections::HashMap;
use std::sync::Arc;
use fabro_types::EventBody;
use chrono::{DateTime, Utc};
use fabro_types::{RunProjection, StageProjection, StageState};
use super::super::{
ApiError, AppState, BilledTokenCounts, BillingByModel, BillingStageRef, EventEnvelope, HashMap,
IntoResponse, Json, ListResponse, ModelBillingTotals, ModelReference, PaginationParams, Path,
Query, RequiredUser, Response, Router, RunBilling, RunBillingStage, RunBillingTotals, RunId,
RunStage, RunStatus, StageState, State, StatusCode, accumulate_model_billing, get,
parse_run_id_path,
ApiError, AppState, BillingByModel, BillingStageRef, IntoResponse, Json, ListResponse,
ModelReference, PaginationParams, Path, Query, RequiredUser, Response, Router, RunBilling,
RunBillingStage, RunBillingTotals, RunId, State, StatusCode, get, parse_run_id_path,
run_stage_from_stage_id,
};
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -16,25 +17,6 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
.route("/runs/{id}/billing", get(get_run_billing))
}
fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState {
let latest = events.iter().rev().find(|envelope| {
envelope.event.node_id.as_deref() == Some(node_id)
&& matches!(
&envelope.event.body,
EventBody::StageRetrying(_)
| EventBody::StageStarted(_)
| EventBody::StageCompleted(_)
| EventBody::StageFailed(_)
)
});
if latest.is_some_and(|e| matches!(&e.event.body, EventBody::StageRetrying(_))) {
StageState::Retrying
} else {
StageState::Running
}
}
async fn list_run_stages(
_auth: RequiredUser,
State(state): State<Arc<AppState>>,
@ -46,81 +28,30 @@ async fn list_run_stages(
Err(response) => return response,
};
// Try live run first.
let (checkpoint, run_is_active) = {
let runs = state.runs.lock().expect("runs lock poisoned");
match runs.get(&id) {
Some(managed_run) => {
let active = !matches!(
managed_run.status,
RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead
);
(managed_run.checkpoint.clone(), active)
}
None => (None, false),
let Ok(run_store) = state.store.open_run_reader(&id).await else {
return ApiError::not_found("Run not found.").into_response();
};
let projection = match run_store.state().await {
Ok(state) => state,
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
// Fall back to stored run.
let (checkpoint, run_is_active) = if checkpoint.is_some() {
(checkpoint, run_is_active)
} else {
match state.store.open_run_reader(&id).await {
Ok(run_store) => match run_store.state().await {
Ok(run_state) => {
let active = run_state.status.is_some_and(|status| !status.is_terminal());
(run_state.checkpoint, active)
}
Err(_) => (None, false),
},
Err(_) => return ApiError::not_found("Run not found.").into_response(),
}
};
let Some(checkpoint) = checkpoint else {
return (
StatusCode::OK,
Json(ListResponse::new(Vec::<RunStage>::new())),
)
.into_response();
};
let events = match state.store.open_run_reader(&id).await {
Ok(run_store) => run_store.list_events().await.unwrap_or_default(),
Err(_) => Vec::new(),
};
let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events);
let mut stages = Vec::new();
for node_id in &checkpoint.completed_nodes {
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
let status = match checkpoint.node_outcomes.get(node_id) {
Some(outcome) => StageState::from(outcome.status),
None => StageState::Succeeded,
};
stages.push(RunStage {
id: node_id.clone(),
name: node_id.clone(),
status,
duration_secs: Some(duration_ms as f64 / 1000.0),
dot_id: Some(node_id.clone()),
});
}
// Add next node as running if the run is still active.
// The checkpoint's current_node is the last *completed* stage; next_node_id
// is the stage that is currently executing.
if let Some(next_id) = &checkpoint.next_node_id {
if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) {
stages.push(RunStage {
id: next_id.clone(),
name: next_id.clone(),
status: active_stage_state_from_events(&events, next_id),
duration_secs: None,
dot_id: Some(next_id.clone()),
});
}
}
let now = Utc::now();
let stages = projection
.iter_stages()
.map(|(stage_id, stage)| {
run_stage_from_stage_id(
stage_id,
stage_id.node_id().to_string(),
stage.effective_state(),
stage.runtime_secs(now),
stage.started_at,
)
})
.collect::<Vec<_>>();
(StatusCode::OK, Json(ListResponse::new(stages))).into_response()
}
@ -137,91 +68,53 @@ async fn get_run_billing(
}
};
let checkpoint = match run_store.state().await {
Ok(state) => state.checkpoint,
let projection = match run_store.state().await {
Ok(state) => state,
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
let Some(checkpoint) = checkpoint else {
let empty = RunBilling {
by_model: Vec::new(),
stages: Vec::new(),
totals: RunBillingTotals {
cache_read_tokens: 0,
cache_write_tokens: 0,
input_tokens: 0,
output_tokens: 0,
reasoning_tokens: 0,
runtime_secs: 0.0,
total_tokens: 0,
total_usd_micros: None,
let rollup = fabro_workflow::billing_rollup_from_projection(&projection);
let by_model = rollup
.by_model
.iter()
.map(|model| BillingByModel {
billing: model.billing.clone(),
model: ModelReference {
id: model.model_id.clone(),
},
};
return (StatusCode::OK, Json(empty)).into_response();
};
stages: model.stages,
})
.collect::<Vec<_>>();
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
let mut by_model_totals = HashMap::<String, ModelBillingTotals>::new();
let mut billed_usages = Vec::new();
let mut runtime_secs = 0.0_f64;
let mut stages = Vec::new();
for node_id in &checkpoint.completed_nodes {
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
runtime_secs += duration_ms as f64 / 1000.0;
let usage = checkpoint
.node_outcomes
.get(node_id)
.and_then(|outcome| outcome.usage.as_ref());
let (billing, model) = if let Some(usage) = usage {
billed_usages.push(usage.clone());
let tokens = usage.tokens();
let billing = BilledTokenCounts {
cache_read_tokens: tokens.cache_read_tokens,
cache_write_tokens: tokens.cache_write_tokens,
input_tokens: tokens.input_tokens,
output_tokens: tokens.output_tokens,
reasoning_tokens: tokens.reasoning_tokens,
total_tokens: tokens.total_tokens(),
total_usd_micros: usage.total_usd_micros,
};
let model_id = usage.model_id().to_string();
accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage);
(billing, Some(ModelReference { id: model_id }))
} else {
(BilledTokenCounts::default(), None)
};
stages.push(RunBillingStage {
billing,
model,
runtime_secs: duration_ms as f64 / 1000.0,
stage: BillingStageRef {
id: node_id.clone(),
name: node_id.clone(),
},
});
}
let totals = BilledTokenCounts::from_billed_usage(&billed_usages);
let by_model = by_model_totals
let rollup_by_node = rollup
.stages
.iter()
.map(|stage| (stage.node_id.as_str(), stage))
.collect::<HashMap<_, _>>();
let live_rows = live_billing_rows(&projection, Utc::now());
let runtime_secs = live_rows.iter().map(|row| row.runtime_secs).sum::<f64>();
let stages = live_rows
.into_iter()
.map(|(model, totals)| BillingByModel {
billing: totals.billing,
model: ModelReference { id: model },
stages: totals.stages,
.map(|row| {
let rollup_stage = rollup_by_node.get(row.node_id.as_str());
RunBillingStage {
billing: rollup_stage
.map(|stage| stage.billing.clone())
.unwrap_or_default(),
model: rollup_stage
.and_then(|stage| stage.model_id.as_ref())
.map(|id| ModelReference { id: id.clone() }),
runtime_secs: row.runtime_secs,
stage: BillingStageRef {
id: row.node_id.clone(),
name: row.node_id,
},
started_at: row.started_at,
state: row.state,
}
})
.collect::<Vec<_>>();
@ -229,16 +122,80 @@ async fn get_run_billing(
by_model,
stages,
totals: RunBillingTotals {
cache_read_tokens: totals.cache_read_tokens,
cache_write_tokens: totals.cache_write_tokens,
input_tokens: totals.input_tokens,
output_tokens: totals.output_tokens,
reasoning_tokens: totals.reasoning_tokens,
cache_read_tokens: rollup.totals.cache_read_tokens,
cache_write_tokens: rollup.totals.cache_write_tokens,
input_tokens: rollup.totals.input_tokens,
output_tokens: rollup.totals.output_tokens,
reasoning_tokens: rollup.totals.reasoning_tokens,
runtime_secs,
total_tokens: totals.total_tokens,
total_usd_micros: totals.total_usd_micros,
total_tokens: rollup.totals.total_tokens,
total_usd_micros: rollup.totals.total_usd_micros,
},
};
(StatusCode::OK, Json(response)).into_response()
}
struct LiveBillingRow {
node_id: String,
runtime_secs: f64,
started_at: Option<DateTime<Utc>>,
state: Option<StageState>,
latest_visit: u32,
}
fn live_billing_rows(projection: &RunProjection, now: DateTime<Utc>) -> Vec<LiveBillingRow> {
let mut row_indices = HashMap::<String, usize>::new();
let mut rows = Vec::<LiveBillingRow>::new();
for (stage_id, stage) in projection.iter_stages() {
let node_id = stage_id.node_id();
if is_exit_stage(projection, node_id) || !stage_has_billing_row(stage) {
continue;
}
let index = *row_indices.entry(node_id.to_string()).or_insert_with(|| {
let index = rows.len();
rows.push(LiveBillingRow {
node_id: node_id.to_string(),
runtime_secs: 0.0,
started_at: None,
state: None,
latest_visit: 0,
});
index
});
let row = &mut rows[index];
row.runtime_secs += billing_runtime_secs(stage, now).unwrap_or(0.0);
if stage_id.visit() >= row.latest_visit {
row.latest_visit = stage_id.visit();
row.started_at = stage.started_at;
row.state = Some(stage.effective_state());
}
}
rows
}
fn billing_runtime_secs(stage: &StageProjection, now: DateTime<Utc>) -> Option<f64> {
stage
.duration_ms
.map(|ms| ms as f64 / 1000.0)
.or_else(|| stage.runtime_secs(now))
}
fn stage_has_billing_row(stage: &StageProjection) -> bool {
stage.completion.is_some()
|| stage.duration_ms.is_some()
|| stage.usage.is_some()
|| stage.started_at.is_some()
|| stage.state.is_some()
}
fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool {
projection
.spec()
.and_then(|spec| spec.graph().nodes.get(node_id))
.is_some_and(|node| node.handler_type() == Some("exit"))
}

311
lib/crates/fabro-server/src/server/handler/events.rs
View file

@ -3,9 +3,10 @@ use std::sync::Arc;
use super::super::{
ApiError, AppState, AppendEventResponse, BroadcastStream, Event, EventBody, EventEnvelope,
EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path,
Query, RequireRunScoped, RequiredUser, Response, Router, RunEvent, RunId, RunStatus, Sse,
State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, mpsc, parse_run_id_path,
redact_jsonl_line, reject_if_archived, update_live_run_from_event,
Query, RequireRunScoped, RequireRunStageScoped, RequiredUser, Response, Router, RunEvent,
RunId, RunStatus, Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get,
mpsc, parse_run_id_path, parse_stage_id_path, redact_jsonl_line, reject_if_archived,
update_live_run_from_event,
};
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -15,11 +16,15 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
"/runs/{id}/events",
get(list_run_events).post(append_run_event),
)
.route(
"/runs/{id}/stages/{stageId}/events",
get(list_run_stage_events),
)
.route("/runs/{id}/attach", get(attach_run_events))
}
#[derive(serde::Deserialize)]
struct EventListParams {
pub(crate) struct EventListParams {
#[serde(default)]
since_seq: Option<u32>,
#[serde(default)]
@ -27,11 +32,11 @@ struct EventListParams {
}
impl EventListParams {
fn since_seq(&self) -> u32 {
pub(crate) fn since_seq(&self) -> u32 {
self.since_seq.unwrap_or(1).max(1)
}
fn limit(&self) -> usize {
pub(crate) fn limit(&self) -> usize {
self.limit.unwrap_or(100).clamp(1, 1000)
}
}
@ -200,6 +205,39 @@ async fn list_run_events(
}
}
async fn list_run_stage_events(
RequireRunStageScoped(id, stage_id): RequireRunStageScoped,
State(state): State<Arc<AppState>>,
Query(params): Query<EventListParams>,
) -> Response {
let stage_id = match parse_stage_id_path(&stage_id) {
Ok(stage_id) => stage_id,
Err(response) => return response,
};
let since_seq = params.since_seq();
let limit = params.limit();
match state.store.open_run_reader(&id).await {
Ok(run_store) => match run_store
.list_events_for_stage_from_with_limit(&stage_id, since_seq, limit)
.await
{
Ok(mut events) => {
let has_more = events.len() > limit;
events.truncate(limit);
Json(PaginatedEventList {
data: events,
meta: PaginationMeta { has_more },
})
.into_response()
}
Err(err) => {
ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response()
}
},
Err(_) => ApiError::not_found("Run not found.").into_response(),
}
}
async fn attach_run_events(
_auth: RequiredUser,
State(state): State<Arc<AppState>>,
@ -342,3 +380,264 @@ fn denied_lifecycle_event_name(body: &EventBody) -> Option<&'static str> {
_ => None,
}
}
#[cfg(test)]
mod stage_events_tests {
use axum::body::{Body, to_bytes};
use axum::http::{Request, StatusCode, header};
use fabro_store::EventPayload;
use fabro_types::RunId;
use serde_json::json;
use tower::ServiceExt;
use crate::test_support::{build_test_router, test_app_state};
fn req_get(uri: &str) -> Request<Body> {
Request::builder()
.method("GET")
.uri(uri)
.body(Body::empty())
.expect("stage events GET request should build")
}
fn make_event(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload {
make_event_with_stage_id(run_id, idx, node_id, None)
}
fn make_event_with_stage_id(
run_id: &RunId,
idx: u32,
node_id: Option<&str>,
stage_id: Option<&str>,
) -> EventPayload {
let mut value = json!({
"id": format!("evt-{idx}"),
"ts": "2026-04-09T12:00:00Z",
"run_id": run_id.to_string(),
"event": "stage.prompt",
"properties": {
"visit": 1,
"text": format!("prompt {idx}"),
},
});
if let Some(node) = node_id {
value
.as_object_mut()
.unwrap()
.insert("node_id".into(), json!(node));
}
if let Some(stage_id) = stage_id {
value
.as_object_mut()
.unwrap()
.insert("stage_id".into(), json!(stage_id));
}
EventPayload::new(value, run_id).expect("event payload should validate")
}
async fn body_json(response: axum::response::Response) -> serde_json::Value {
let bytes = to_bytes(response.into_body(), usize::MAX)
.await
.expect("response body should fit in memory");
serde_json::from_slice(&bytes).expect("response body should be valid JSON")
}
async fn seed_run_with_mixed_events() -> (RunId, axum::Router) {
let state = test_app_state();
let app = build_test_router(state.clone());
let run_id = RunId::new();
let run_store = state
.store_ref()
.create_run(&run_id)
.await
.expect("test run should be creatable");
// Seed 200 unrelated 'beta' events first so any node-blind
// truncation would lose the sparse 'alpha' tail. Then 3 'alpha'
// events past seq 100, plus a couple with no node_id at all.
for idx in 1..=200_u32 {
run_store
.append_event(&make_event(&run_id, idx, Some("beta")))
.await
.expect("append should succeed");
}
run_store
.append_event(&make_event(&run_id, 201, None))
.await
.expect("append should succeed");
for idx in 202..=204_u32 {
run_store
.append_event(&make_event(&run_id, idx, Some("alpha")))
.await
.expect("append should succeed");
}
(run_id, app)
}
#[tokio::test]
async fn returns_only_matching_node_events_in_seq_order() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha@1/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let data = body["data"].as_array().expect("data is array");
let seqs: Vec<u64> = data.iter().map(|e| e["seq"].as_u64().unwrap()).collect();
assert_eq!(seqs, vec![202, 203, 204]);
assert_eq!(body["meta"]["has_more"], false);
}
#[tokio::test]
async fn since_seq_filters_to_events_with_seq_at_least_k() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha@1/events?since_seq=203"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let seqs: Vec<u64> = body["data"]
.as_array()
.unwrap()
.iter()
.map(|e| e["seq"].as_u64().unwrap())
.collect();
assert_eq!(seqs, vec![203, 204]);
}
#[tokio::test]
async fn limit_one_returns_first_envelope_with_has_more_true() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha@1/events?limit=1"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let data = body["data"].as_array().unwrap();
assert_eq!(data.len(), 1);
assert_eq!(data[0]["seq"].as_u64().unwrap(), 202);
assert_eq!(body["meta"]["has_more"], true);
}
#[tokio::test]
async fn unknown_stage_in_existing_run_returns_empty_list_with_no_more() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/unknown-stage@1/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
assert_eq!(body["data"].as_array().unwrap().len(), 0);
assert_eq!(body["meta"]["has_more"], false);
}
#[tokio::test]
async fn missing_run_returns_404_with_run_not_found() {
let app = build_test_router(test_app_state());
// A syntactically valid RunId that the store has never seen, so
// `parse_run_id_path` succeeds but `open_run_reader` fails — that
// exercises the handler's not-found branch rather than the path
// parser's 400 branch.
let absent = RunId::new();
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{absent}/stages/alpha@1/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
let body = body_json(response).await;
let detail = body["errors"][0]["detail"]
.as_str()
.expect("error detail string");
assert!(
detail.contains("Run not found."),
"unexpected error body: {body}"
);
}
#[tokio::test]
async fn unauthenticated_request_is_rejected() {
let state = test_app_state();
// Bypass `build_test_router`'s auto-injected bearer token by
// building the raw router directly. The principal middleware sees
// a missing Authorization header and the extractor enforces auth.
let app = crate::server::build_router(state, crate::test_support::test_auth_mode());
let run_id = RunId::new();
let request = Request::builder()
.method("GET")
.uri(format!("/api/v1/runs/{run_id}/stages/alpha@1/events"))
.header(header::ACCEPT, "application/json")
.body(Body::empty())
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn returns_only_requested_visit_when_stage_id_is_present() {
let state = test_app_state();
let app = build_test_router(state.clone());
let run_id = RunId::new();
let run_store = state
.store_ref()
.create_run(&run_id)
.await
.expect("test run should be creatable");
run_store
.append_event(&make_event_with_stage_id(
&run_id,
1,
Some("verify"),
Some("verify@1"),
))
.await
.expect("append should succeed");
run_store
.append_event(&make_event_with_stage_id(
&run_id,
2,
Some("verify"),
Some("verify@2"),
))
.await
.expect("append should succeed");
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/verify@2/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let seqs: Vec<u64> = body["data"]
.as_array()
.unwrap()
.iter()
.map(|e| e["seq"].as_u64().unwrap())
.collect();
assert_eq!(seqs, vec![2]);
}
}

5
lib/crates/fabro-server/src/server/handler/mod.rs
View file

@ -57,8 +57,8 @@ pub(super) fn demo_routes() -> Router<Arc<AppState>> {
.route("/runs/{id}/artifacts", get(demo::list_run_artifacts_stub))
.route("/runs/{id}/files", get(demo::list_run_files_stub))
.route(
"/runs/{id}/stages/{stageId}/turns",
get(demo::get_stage_turns),
"/runs/{id}/stages/{stageId}/events",
get(demo::get_stage_events),
)
.route(
"/runs/{id}/stages/{stageId}/artifacts",
@ -113,7 +113,6 @@ pub(super) fn demo_routes() -> Router<Arc<AppState>> {
pub(super) fn real_routes() -> Router<Arc<AppState>> {
Router::new()
.route("/runs/{id}/stages/{stageId}/turns", get(not_implemented))
.route("/runs/{id}/steer", post(not_implemented))
.route("/workflows", get(not_implemented))
.route("/workflows/{name}", get(not_implemented))

52
lib/crates/fabro-server/src/server/handler/runs.rs
View file

@ -10,7 +10,9 @@ use axum::{Json, Router};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use fabro_api::types::{RunManifest, RunStatusResponse, SubmitAnswerRequest};
use fabro_api::types::{
BoardColumn, BoardColumnDefinition, RunManifest, RunStatusResponse, SubmitAnswerRequest,
};
use fabro_config::Storage;
use fabro_interview::AnswerSubmission;
use fabro_types::{
@ -82,25 +84,45 @@ impl ListRunsParams {
}
}
fn board_column(status: RunStatus) -> Option<&'static str> {
fn board_column(status: RunStatus) -> Option<BoardColumn> {
match status {
RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting => Some("initializing"),
RunStatus::Running | RunStatus::Paused { .. } => Some("running"),
RunStatus::Blocked { .. } => Some("blocked"),
RunStatus::Succeeded { .. } => Some("succeeded"),
RunStatus::Failed { .. } | RunStatus::Dead => Some("failed"),
RunStatus::Submitted | RunStatus::Queued => Some(BoardColumn::Queued),
RunStatus::Starting => Some(BoardColumn::Initializing),
RunStatus::Running | RunStatus::Paused { .. } => Some(BoardColumn::Running),
RunStatus::Blocked { .. } => Some(BoardColumn::Blocked),
RunStatus::Succeeded { .. } => Some(BoardColumn::Succeeded),
RunStatus::Failed { .. } | RunStatus::Dead => Some(BoardColumn::Failed),
RunStatus::Removing | RunStatus::Archived { .. } => None,
}
}
pub(crate) fn board_columns() -> serde_json::Value {
serde_json::json!([
{"id": "initializing", "name": "Initializing"},
{"id": "running", "name": "Running"},
{"id": "blocked", "name": "Blocked"},
{"id": "succeeded", "name": "Succeeded"},
{"id": "failed", "name": "Failed"},
])
pub(crate) fn board_columns() -> Vec<BoardColumnDefinition> {
vec![
BoardColumnDefinition {
id: BoardColumn::Queued,
name: "Queued".into(),
},
BoardColumnDefinition {
id: BoardColumn::Initializing,
name: "Initializing".into(),
},
BoardColumnDefinition {
id: BoardColumn::Running,
name: "Running".into(),
},
BoardColumnDefinition {
id: BoardColumn::Blocked,
name: "Blocked".into(),
},
BoardColumnDefinition {
id: BoardColumn::Succeeded,
name: "Succeeded".into(),
},
BoardColumnDefinition {
id: BoardColumn::Failed,
name: "Failed".into(),
},
]
}
async fn board_run_metadata(

999
lib/crates/fabro-server/src/server/tests.rs
View file

File diff suppressed because it is too large Load diff

75
lib/crates/fabro-server/tests/it/event_pagination.rs Normal file
View file

@ -0,0 +1,75 @@
//! Cursor-pagination tests for the per-stage events endpoint (demo mode).
//!
//! The stage-events route uses `since_seq=` + `limit=` (cursor-based) instead
//! of the offset-based `page[limit]/page[offset]` pagination used by other
//! list endpoints, so it gets its own test rather than living in the generic
//! offset-shape matrix.
#![allow(
clippy::absolute_paths,
reason = "This test module prefers explicit type paths over extra imports."
)]
use axum::body::Body;
use axum::http::{Request, StatusCode};
use tower::ServiceExt;
use super::helpers::{response_json, test_app_state};
async fn get_json(app: &axum::Router, uri: &str) -> serde_json::Value {
let req = Request::builder()
.method("GET")
.uri(uri)
.header("x-fabro-demo", "1")
.body(Body::empty())
.expect("event pagination request should build");
let response = app.clone().oneshot(req).await.unwrap();
response_json(response, StatusCode::OK, format!("GET {uri}")).await
}
#[tokio::test]
async fn demo_stage_events_default_returns_all_fixture_events_with_no_more() {
let app = fabro_server::test_support::build_test_router(test_app_state());
let body = get_json(&app, "/api/v1/runs/run-1/stages/detect-drift@1/events").await;
let data = body["data"].as_array().expect("data is an array");
assert_eq!(data.len(), 7, "all seven fixture events should be returned");
assert_eq!(body["meta"]["has_more"], false);
}
#[tokio::test]
async fn demo_stage_events_limit_one_signals_has_more() {
let app = fabro_server::test_support::build_test_router(test_app_state());
let body = get_json(
&app,
"/api/v1/runs/run-1/stages/detect-drift@1/events?limit=1",
)
.await;
let data = body["data"].as_array().expect("data is an array");
assert_eq!(data.len(), 1);
assert_eq!(body["meta"]["has_more"], true);
}
#[tokio::test]
async fn demo_stage_events_since_seq_filters_out_earlier_events() {
let app = fabro_server::test_support::build_test_router(test_app_state());
// The fixture seqs are 1..=7. since_seq=4 should skip the first three.
let body = get_json(
&app,
"/api/v1/runs/run-1/stages/detect-drift@1/events?since_seq=4",
)
.await;
let data = body["data"].as_array().expect("data is an array");
assert_eq!(data.len(), 4);
let seqs: Vec<u64> = data
.iter()
.map(|envelope| envelope["seq"].as_u64().expect("seq is a number"))
.collect();
assert_eq!(seqs, vec![4, 5, 6, 7]);
assert_eq!(body["meta"]["has_more"], false);
}

1
lib/crates/fabro-server/tests/it/main.rs
View file

@ -4,6 +4,7 @@
)]
mod api;
mod event_pagination;
mod helpers;
mod openapi_conformance;
mod pagination;

4
lib/crates/fabro-server/tests/it/pagination.rs
View file

@ -56,10 +56,6 @@ const ENDPOINTS: &[PaginatedEndpoint] = &[
path: "/api/v1/models",
name: "listModels",
},
PaginatedEndpoint {
path: "/api/v1/runs/run-1/stages/detect-drift/turns",
name: "listStageTurns",
},
PaginatedEndpoint {
path: "/api/v1/runs/run-1/questions",
name: "listRunQuestions",

7
lib/crates/fabro-store/src/artifact_store.rs
View file

@ -297,8 +297,13 @@ fn decode_artifact_location(
))
})?;
let (retry, filename) = decode_retry_and_filename(location, &mut parts)?;
let stage_id = StageId::try_new(node_id, visit).map_err(|err| {
Error::Other(format!(
"artifact location {location} has an invalid stage id: {err}"
))
})?;
Ok(NodeArtifact {
node: StageId::new(node_id, visit),
node: stage_id,
retry,
filename,
size,

563
lib/crates/fabro-store/src/run_state.rs
View file

@ -10,7 +10,8 @@ use fabro_types::{
BilledModelUsage, Checkpoint, CommandTermination, Conclusion, EventBody, FailureSignature,
InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction,
RunEvent, RunId, RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion,
StageOutcome, StageProjection, StartRecord, TerminalStatus, first_event_seq,
StageId, StageOutcome, StageProjection, StageState, StartRecord, TerminalStatus,
first_event_seq,
};
use fabro_util::error::render_with_causes;
use serde_json::Value;
@ -290,59 +291,76 @@ impl RunProjectionReducer for RunProjection {
let Some(stage_id) = stored.stage_id.as_ref() else {
return Ok(());
};
self.stage_entry(
let stage = self.stage_entry(
stage_id.node_id(),
stage_id.visit(),
first_event_seq(event.seq),
);
stage.begin_attempt(ts);
}
EventBody::StageRetrying(_) => {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.state = Some(StageState::Retrying);
}
EventBody::StagePrompt(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.prompt = Some(props.text.clone());
stage.provider_used = provider_used_from_prompt(props);
}
EventBody::PromptCompleted(props) => {
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.response = Some(props.response.clone());
}
EventBody::StageCompleted(props) => {
let Some(node_id) = stored.node_id.as_deref() else {
return Ok(());
};
let visit = stage_visit(node_id, props.node_visits.as_ref(), self).unwrap_or(1);
let response = props.response.clone();
let outcome = stage_outcome_from_props(props);
let completion = stage_completion_from_outcome(&outcome, ts);
let stage = self.stage_entry(node_id, visit, first_event_seq(event.seq));
let Some(stage) =
stage_at_completed_visit(self, stored, props.node_visits.as_ref(), event.seq)
else {
return Ok(());
};
stage.response = response;
stage.completion = Some(completion);
stage.duration_ms = Some(props.duration_ms);
stage.usage.clone_from(&props.billing);
stage.state = Some(StageState::from(outcome.status));
}
EventBody::StageFailed(props) => {
let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone());
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
let outcome = StageOutcome::Failed {
retry_requested: props.will_retry,
};
stage.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: false,
},
outcome,
notes: None,
failure_reason,
timestamp: ts,
});
stage.duration_ms = Some(props.duration_ms);
stage.usage.clone_from(&props.billing);
stage.state = Some(StageState::from(outcome));
}
EventBody::AgentSessionStarted(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.provider_used = Some(provider_used_from_agent_session_started(props));
}
EventBody::AgentCliStarted(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.provider_used = Some(provider_used_from_agent_cli_started(props));
@ -351,7 +369,7 @@ impl RunProjectionReducer for RunProjection {
let script_invocation = serde_json::to_value(props).map_err(|err| {
Error::InvalidEvent(format!("invalid command.started payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.script_invocation = Some(script_invocation);
@ -360,7 +378,7 @@ impl RunProjectionReducer for RunProjection {
let script_timing = serde_json::to_value(props).map_err(|err| {
Error::InvalidEvent(format!("invalid command.completed payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.stdout = Some(props.stdout.clone());
@ -412,7 +430,7 @@ impl RunProjectionReducer for RunProjection {
let parallel_results = serde_json::to_value(&props.results).map_err(|err| {
Error::InvalidEvent(format!("invalid parallel.completed payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.parallel_results = Some(parallel_results);
@ -430,6 +448,9 @@ fn stage_at_visit<'a>(
visit: u32,
seq: u32,
) -> Option<&'a mut StageProjection> {
if visit == 0 {
return None;
}
let node_id = stored.node_id.as_deref()?;
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
@ -444,6 +465,51 @@ fn stage_at_current_visit<'a>(
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
fn stage_at_stored_stage_id<'a>(
state: &'a mut RunProjection,
stage_id: &StageId,
seq: u32,
) -> &'a mut StageProjection {
state.stage_entry(stage_id.node_id(), stage_id.visit(), first_event_seq(seq))
}
fn stage_at_stored_or_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
visit: u32,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
stage_at_visit(state, stored, visit, seq)
}
fn stage_at_stored_or_current_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
stage_at_current_visit(state, stored, seq)
}
fn stage_at_completed_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
node_visits: Option<&BTreeMap<String, usize>>,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
let node_id = stored.node_id.as_deref()?;
let visit = stage_visit(node_id, node_visits, state).unwrap_or(1);
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> RunSummary {
let workflow_name = state.spec.as_ref().map(|spec| {
if spec.graph.name.is_empty() {
@ -565,6 +631,7 @@ fn stage_visit(
node_visits
.and_then(|visits| visits.get(node_id).copied())
.and_then(|visit| u32::try_from(visit).ok())
.filter(|visit| *visit > 0)
.or_else(|| state.current_visit_for(node_id))
}
@ -666,12 +733,14 @@ mod tests {
use fabro_types::run_event::{
AgentCliCancelledProps, AgentCliCompletedProps, AgentCliTimedOutProps,
CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps,
RunControlEffectProps, StagePromptProps, StageStartedProps,
RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps,
StageRetryingProps, StageStartedProps,
};
use fabro_types::{
BlockedReason, Checkpoint, CommandTermination, EventBody, FailureReason, Outcome,
QuestionType, RunBlobId, RunControlAction, RunEvent, RunStatus, StageOutcome,
SuccessReason, TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, BlockedReason, Checkpoint, CommandTermination, EventBody,
FailureCategory, FailureDetail, FailureReason, Outcome, QuestionType, RunBlobId,
RunControlAction, RunEvent, RunStatus, StageOutcome, StageState, SuccessReason,
TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
@ -704,6 +773,32 @@ mod tests {
event
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
fn usage_json(usage: &BilledModelUsage) -> serde_json::Value {
serde_json::to_value(usage).unwrap()
}
fn test_raw_event(
seq: u32,
event: &str,
@ -1014,6 +1109,210 @@ mod tests {
);
}
#[test]
fn stage_completed_event_captures_duration_and_usage_per_visit() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 123, 45);
state
.apply_event(&test_event(
3,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms: 789,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("done".to_string()),
attempt: 1,
max_attempts: 1,
}),
Some("build"),
))
.unwrap();
let stage = state.stage(&StageId::new("build", 1)).unwrap();
assert_eq!(stage.duration_ms, Some(789));
assert_eq!(stage.usage.as_ref(), Some(&usage));
}
#[test]
fn stage_failed_event_captures_duration_and_usage_per_visit() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
let usage = test_usage("gpt-5.2", 321, 54);
state
.apply_event(&test_stage_event(
2,
EventBody::StageStarted(StageStartedProps {
index: 0,
handler_type: "agent".to_string(),
attempt: 1,
max_attempts: 1,
}),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_raw_event(
3,
"stage.failed",
&json!({
"index": 0,
"failure": {
"message": "provider failed",
"failure_class": "transient_infra"
},
"will_retry": false,
"duration_ms": 654,
"billing": usage_json(&usage)
}),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(654));
assert_eq!(stage.usage.as_ref(), Some(&usage));
}
#[test]
fn two_visits_of_one_node_retain_distinct_usage() {
let mut state = RunProjection::default();
let first_usage = test_usage("gpt-5.2", 100, 10);
let second_usage = test_usage("gpt-5.2", 200, 20);
for (seq, visit, duration_ms, usage) in [
(3, 1usize, 111, first_usage.clone()),
(4, 2usize, 222, second_usage.clone()),
] {
state
.apply_event(&test_event(
seq,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: Some(BTreeMap::from([("build".to_string(), visit)])),
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
}),
Some("build"),
))
.unwrap();
}
let first_stage = state.stage(&StageId::new("build", 1)).unwrap();
let second_stage = state.stage(&StageId::new("build", 2)).unwrap();
assert_eq!(first_stage.duration_ms, Some(111));
assert_eq!(first_stage.usage.as_ref(), Some(&first_usage));
assert_eq!(second_stage.duration_ms, Some(222));
assert_eq!(second_stage.usage.as_ref(), Some(&second_usage));
}
#[test]
fn stage_completed_prefers_stored_stage_id_over_legacy_node_visits() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 300, 30);
let scoped_stage_id = StageId::new("build", 2);
state
.apply_event(&test_stage_event(
3,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms: 333,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: Some(BTreeMap::from([("build".to_string(), 1usize)])),
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("done".to_string()),
attempt: 1,
max_attempts: 1,
}),
scoped_stage_id.clone(),
))
.unwrap();
assert!(
state.stage(&StageId::new("build", 1)).is_none(),
"legacy node_visits must not override stored stage_id"
);
let stage = state.stage(&scoped_stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(333));
assert_eq!(stage.usage.as_ref(), Some(&usage));
assert_eq!(stage.response.as_deref(), Some("done"));
}
#[test]
fn stage_failed_prefers_stored_stage_id_and_preserves_retry_request() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 400, 40);
let scoped_stage_id = StageId::new("build", 2);
state
.apply_event(&test_stage_event(
3,
EventBody::StageFailed(StageFailedProps {
index: 0,
failure: Some(fabro_types::FailureDetail::new(
"try again",
fabro_types::FailureCategory::TransientInfra,
)),
will_retry: true,
duration_ms: 444,
billing: Some(usage.clone()),
}),
scoped_stage_id.clone(),
))
.unwrap();
assert!(
state.stage(&StageId::new("build", 1)).is_none(),
"current-visit fallback must not override stored stage_id"
);
let stage = state.stage(&scoped_stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(444));
assert_eq!(stage.usage.as_ref(), Some(&usage));
let completion = stage.completion.as_ref().unwrap();
assert_eq!(completion.outcome, StageOutcome::Failed {
retry_requested: true,
});
assert_eq!(completion.failure_reason.as_deref(), Some("try again"));
}
#[test]
fn checkpoint_completed_creates_projection_entry_for_skipped_stage() {
let mut state = RunProjection::default();
@ -1641,4 +1940,224 @@ mod tests {
);
assert_eq!(state.status_updated_at, updated_at);
}
fn started_props() -> StageStartedProps {
StageStartedProps {
index: 0,
handler_type: "agent".to_string(),
attempt: 1,
max_attempts: 3,
}
}
fn failed_props(duration_ms: u64, will_retry: bool) -> StageFailedProps {
StageFailedProps {
index: 0,
failure: Some(FailureDetail::new("boom", FailureCategory::TransientInfra)),
will_retry,
duration_ms,
billing: None,
}
}
fn retrying_props() -> StageRetryingProps {
StageRetryingProps {
index: 0,
attempt: 2,
max_attempts: 3,
delay_ms: 0,
}
}
fn completed_props(duration_ms: u64, status: StageOutcome) -> StageCompletedProps {
StageCompletedProps {
index: 0,
duration_ms,
status,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 3,
}
}
fn billed_usage() -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": "gpt-test"
},
"tokens": {
"input_tokens": 10,
"output_tokens": 5,
"reasoning_tokens": 2,
"cache_read_tokens": 3,
"cache_write_tokens": 4
}
},
"facts": { "provider": "open_ai" }
},
"total_usd_micros": 123
}))
.expect("billing fixture should deserialize")
}
#[test]
fn stage_started_records_started_at_and_running_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
3,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.state, Some(StageState::Running));
assert!(stage.started_at.is_some());
assert_eq!(stage.effective_state(), StageState::Running);
}
#[test]
fn stage_completed_records_duration_usage_and_terminal_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
let usage = billed_usage();
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
let mut props = completed_props(42, StageOutcome::Succeeded);
props.billing = Some(usage.clone());
state
.apply_event(&test_event(
2,
EventBody::StageCompleted(props),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(42));
assert_eq!(stage.usage.as_ref(), Some(&usage));
assert_eq!(stage.state, Some(StageState::Succeeded));
assert_eq!(stage.effective_state(), StageState::Succeeded);
}
#[test]
fn stage_failed_records_duration_and_failed_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_event(
2,
EventBody::StageFailed(failed_props(10, false)),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(10));
assert_eq!(stage.state, Some(StageState::Failed));
}
#[test]
fn stage_retrying_sets_retrying_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_event(
2,
EventBody::StageFailed(failed_props(10, true)),
Some("build"),
))
.unwrap();
state
.apply_event(&test_event(
3,
EventBody::StageRetrying(retrying_props()),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.state, Some(StageState::Retrying));
}
#[test]
fn stage_started_after_retrying_returns_to_running_and_resets_attempt_data() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_event(
2,
EventBody::StageFailed(failed_props(10, true)),
Some("build"),
))
.unwrap();
state
.apply_event(&test_event(
3,
EventBody::StageRetrying(retrying_props()),
Some("build"),
))
.unwrap();
state
.apply_event(&test_stage_event(
4,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.state, Some(StageState::Running));
// Prior attempt's terminal data must not leak into the new attempt.
assert!(stage.completion.is_none());
assert_eq!(stage.duration_ms, None);
}
}

285
lib/crates/fabro-store/src/slate/run_store.rs
View file

@ -12,7 +12,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream;
use super::blob_store::BlobStore;
use crate::run_state::{EventProjectionCache, RunProjectionReducer, build_summary};
use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, keys};
use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, StageId, keys};
const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024;
#[derive(Clone)]
@ -213,6 +213,30 @@ impl RunDatabase {
list_events_from_with_limit(&self.inner.db, &self.inner.run_id, start_seq, limit).await
}
/// Returns up to `limit + 1` events for the given stage visit,
/// starting at `start_seq`. The `+1` lets callers compute `has_more`.
///
/// Implementation note: scans the unbounded run-event prefix and
/// filters by stage identity *before* applying `limit`, so a stage with
/// matches sparsely scattered late in the event log still returns its
/// full slice (no premature truncation from a generic `limit`-bounded
/// scan).
pub async fn list_events_for_stage_from_with_limit(
&self,
stage_id: &StageId,
start_seq: u32,
limit: usize,
) -> Result<Vec<EventEnvelope>> {
list_events_for_stage_from_with_limit(
&self.inner.db,
&self.inner.run_id,
stage_id,
start_seq,
limit,
)
.await
}
pub fn watch_events_from(
&self,
seq: u32,
@ -348,6 +372,74 @@ where
Ok(events)
}
async fn list_events_for_stage_from_with_limit<R>(
db: &R,
run_id: &RunId,
stage_id: &StageId,
start_seq: u32,
limit: usize,
) -> Result<Vec<EventEnvelope>>
where
R: DbRead + Sync,
{
// Unbounded scan first: filtering by stage identity with a generic
// limit-bounded scan would silently drop matches whenever the stage's
// events are sparse late in the event log.
//
// We probe just the stage identity fields with a small partial deserialize and
// only run the full `RunEvent` parse on matches. Most events in a run
// belong to other nodes, so this avoids deserializing large payloads
// (`agent.tool.completed.output`, `agent.message.text`, …) we'd discard.
#[derive(serde::Deserialize)]
struct StageIdProbe<'a> {
#[serde(default, borrow)]
stage_id: Option<&'a str>,
#[serde(default, borrow)]
node_id: Option<&'a str>,
}
let stage_id_string = stage_id.to_string();
let max_events = limit.saturating_add(1);
let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?;
let mut events: Vec<EventEnvelope> = Vec::new();
while let Some(entry) = iter.next().await? {
let key = key_to_string(&entry.key)?;
let Some(seq) = keys::parse_event_seq(&key) else {
continue;
};
if seq < start_seq {
continue;
}
let probe: StageIdProbe = serde_json::from_slice(&entry.value)?;
let matches_stage_id = probe.stage_id == Some(stage_id_string.as_str());
let matches_legacy_node_id = probe.stage_id.is_none()
&& stage_id.visit() == 1
&& probe.node_id == Some(stage_id.node_id());
if !matches_stage_id && !matches_legacy_node_id {
continue;
}
let event: RunEvent = serde_json::from_slice(&entry.value)?;
let envelope = EventEnvelope { seq, event };
if events.len() < max_events {
events.push(envelope);
continue;
}
if let Some((max_index, max_seq)) = events
.iter()
.enumerate()
.max_by_key(|(_, existing)| existing.seq)
.map(|(index, existing)| (index, existing.seq))
{
if seq < max_seq {
events[max_index] = envelope;
}
}
}
events.sort_by_key(|event| event.seq);
Ok(events)
}
async fn list_blobs<R>(db: &R) -> Result<Vec<RunBlobId>>
where
R: DbRead + Sync,
@ -375,9 +467,12 @@ mod tests {
use std::sync::Arc;
use std::time::Duration;
use fabro_types::{RunId, StageId};
use object_store::memory::InMemory;
use serde_json::json;
use crate::{Database, EventPayload};
use crate::Database;
#[tokio::test]
async fn list_blobs_reads_global_cas_namespace() {
let object_store = Arc::new(InMemory::new());
@ -394,4 +489,190 @@ mod tests {
assert_eq!(blob_ids, vec![first_id, second_id]);
}
fn stage_prompt_payload(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload {
stage_prompt_payload_for_stage(run_id, idx, node_id, None)
}
fn stage_prompt_payload_for_stage(
run_id: &RunId,
idx: u32,
node_id: Option<&str>,
stage_id: Option<&StageId>,
) -> EventPayload {
let mut value = json!({
"id": format!("evt-{idx}"),
"ts": "2026-04-09T12:00:00Z",
"run_id": run_id.to_string(),
"event": "stage.prompt",
"properties": {
"visit": 1,
"text": format!("prompt {idx}"),
},
});
if let Some(node_id) = node_id {
value
.as_object_mut()
.unwrap()
.insert("node_id".into(), json!(node_id));
}
if let Some(stage_id) = stage_id {
value
.as_object_mut()
.unwrap()
.insert("stage_id".into(), json!(stage_id.to_string()));
}
EventPayload::new(value, run_id).unwrap()
}
async fn fresh_run() -> super::RunDatabase {
let object_store = Arc::new(InMemory::new());
let store = Database::new(object_store, "", Duration::from_millis(1), None);
let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap();
store.create_run(&run_id).await.unwrap()
}
#[tokio::test]
async fn list_events_for_stage_returns_only_matching_events_in_seq_order() {
let run = fresh_run().await;
let run_id = run.run_id();
run.append_event(&stage_prompt_payload(&run_id, 1, Some("alpha")))
.await
.unwrap();
run.append_event(&stage_prompt_payload(&run_id, 2, Some("beta")))
.await
.unwrap();
run.append_event(&stage_prompt_payload(&run_id, 3, Some("alpha")))
.await
.unwrap();
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![1, 3]);
}
#[tokio::test]
async fn list_events_for_stage_skips_events_with_no_stage_identity() {
let run = fresh_run().await;
let run_id = run.run_id();
run.append_event(&stage_prompt_payload(&run_id, 1, None))
.await
.unwrap();
run.append_event(&stage_prompt_payload(&run_id, 2, Some("alpha")))
.await
.unwrap();
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![2]);
}
#[tokio::test]
async fn list_events_for_stage_paginates_via_start_seq_on_filtered_slice() {
let run = fresh_run().await;
let run_id = run.run_id();
for idx in 1..=5 {
let node = if idx % 2 == 0 { "beta" } else { "alpha" };
run.append_event(&stage_prompt_payload(&run_id, idx, Some(node)))
.await
.unwrap();
}
// alpha events live at seqs 1, 3, 5. Start at seq=2 should skip seq=1.
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 2, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![3, 5]);
}
#[tokio::test]
async fn list_events_for_stage_walks_past_unrelated_events_for_sparse_matches() {
let run = fresh_run().await;
let run_id = run.run_id();
// 200 unrelated events first.
for idx in 1..=200 {
run.append_event(&stage_prompt_payload(&run_id, idx, Some("noise")))
.await
.unwrap();
}
// Then 3 sparse "alpha" events at the tail.
for idx in 201..=203 {
run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha")))
.await
.unwrap();
}
// limit smaller than the number of unrelated events would have
// truncated the upstream scan if we had post-filtered.
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 5)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![201, 202, 203]);
}
#[tokio::test]
async fn list_events_for_stage_returns_limit_plus_one_for_has_more_signal() {
let run = fresh_run().await;
let run_id = run.run_id();
for idx in 1..=5 {
run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha")))
.await
.unwrap();
}
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 2)
.await
.unwrap();
// With limit=2, we expect up to limit+1 = 3 envelopes so the
// caller can compute has_more.
assert_eq!(events.len(), 3);
}
#[tokio::test]
async fn list_events_for_stage_prefers_stage_id_over_node_id() {
let run = fresh_run().await;
let run_id = run.run_id();
let first_visit = StageId::new("verify", 1);
let second_visit = StageId::new("verify", 2);
run.append_event(&stage_prompt_payload_for_stage(
&run_id,
1,
Some("verify"),
Some(&first_visit),
))
.await
.unwrap();
run.append_event(&stage_prompt_payload_for_stage(
&run_id,
2,
Some("verify"),
Some(&second_visit),
))
.await
.unwrap();
let events = run
.list_events_for_stage_from_with_limit(&second_visit, 1, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![2]);
}
}

34
lib/crates/fabro-store/tests/serializable_projection.rs
View file

@ -5,8 +5,8 @@ use fabro_store::{RunProjection, SerializableProjection, StageId};
use fabro_types::graph::Graph;
use fabro_types::run::RunSpec;
use fabro_types::{
Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, StartRecord,
TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome,
StartRecord, TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
@ -52,6 +52,28 @@ fn sample_checkpoint() -> Checkpoint {
}
}
fn sample_usage() -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": "gpt-5.2"
},
"tokens": {
"input_tokens": 123,
"output_tokens": 45
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": 168
}))
.expect("sample usage should deserialize")
}
#[test]
fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
let stage_id = StageId::new("build", 2);
@ -94,11 +116,17 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
stage.script_invocation = Some(json!({ "command": "cargo test" }));
stage.script_timing = Some(json!({ "duration_ms": 10 }));
stage.parallel_results = Some(json!([{ "stage": "fanout@1" }]));
stage.duration_ms = Some(1234);
stage.usage = Some(sample_usage());
stage.stdout = Some("stdout".to_string());
stage.stderr = Some("stderr".to_string());
let serialized = serde_json::to_value(SerializableProjection(&projection))
.expect("projection should serialize");
assert!(
serialized["stages"]["build@2"].get("usage").is_none(),
"stage usage is server-internal and should not be serialized"
);
let round_tripped: RunProjection =
serde_json::from_value(serialized).expect("serialized projection should deserialize");
let node = round_tripped.stage(&stage_id).expect("node should remain");
@ -138,6 +166,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
node.parallel_results,
Some(json!([{ "stage": "fanout@1" }]))
);
assert_eq!(node.duration_ms, Some(1234));
assert_eq!(node.usage, None);
}
#[test]

4
lib/crates/fabro-types/src/lib.rs
View file

@ -72,7 +72,7 @@ pub use run::{
pub use run_blob_id::RunBlobId;
pub use run_event::{
EventBody, ExecOutputTail, InterviewOption, MetadataSnapshotFailureKind, MetadataSnapshotPhase,
RunEvent, RunNoticeLevel,
RunEvent, RunNoticeCode, RunNoticeLevel,
};
pub use run_id::{RunId, fixtures};
pub use run_projection::{PendingInterviewRecord, RunProjection, StageProjection, first_event_seq};
@ -80,7 +80,7 @@ pub use run_summary::RunSummary;
pub use sandbox_record::SandboxRecord;
pub use secret::{SecretMetadata, SecretType};
pub use stage_completion::StageCompletion;
pub use stage_id::{ParallelBranchId, StageId};
pub use stage_id::{InvalidStageVisit, ParallelBranchId, StageId};
pub use start::StartRecord;
pub use status::{
BlockedReason, FailureReason, InvalidTransition, ParseFailureReasonError,

25
lib/crates/fabro-types/src/outcome.rs
View file

@ -138,12 +138,32 @@ impl From<StageOutcome> for StageState {
match outcome {
StageOutcome::Succeeded => Self::Succeeded,
StageOutcome::PartiallySucceeded => Self::PartiallySucceeded,
StageOutcome::Failed { .. } => Self::Failed,
StageOutcome::Failed {
retry_requested: true,
} => Self::Retrying,
StageOutcome::Failed {
retry_requested: false,
} => Self::Failed,
StageOutcome::Skipped => Self::Skipped,
}
}
}
#[cfg(test)]
mod stage_state_tests {
use super::{StageOutcome, StageState};
#[test]
fn retry_requested_failure_projects_as_retrying() {
assert_eq!(
StageState::from(StageOutcome::Failed {
retry_requested: true,
}),
StageState::Retrying
);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FailureCategory {
@ -340,9 +360,10 @@ mod tests {
StageState::from(StageOutcome::Failed {
retry_requested: true,
}),
StageState::Failed
StageState::Retrying
);
assert!(StageState::Cancelled.is_terminal());
assert!(!StageState::Retrying.is_terminal());
assert!(!StageState::Running.is_terminal());
}
}

53
lib/crates/fabro-types/src/run_event/infra.rs
View file

@ -1,17 +1,48 @@
use serde::{Deserialize, Serialize};
/// Legacy `run.notice` codes paired with the new `metadata.snapshot.failed`
/// event for backward compatibility. Display layers suppress these so the
/// typed event renders without a duplicate raw warning.
pub const NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED: &str = "checkpoint_metadata_write_failed";
pub const NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED: &str = "checkpoint_metadata_push_failed";
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
Hash,
Serialize,
Deserialize,
strum::Display,
strum::EnumString,
strum::IntoStaticStr,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum RunNoticeCode {
ArtifactCollectionFailed,
ArtifactOffloadFailed,
ArtifactSyncFailed,
ArtifactUploadFailed,
CheckpointMetadataDegraded,
CheckpointMetadataPushFailed,
CheckpointMetadataWriteFailed,
DirtyWorktree,
GitDiffFailed,
GitPushFailed,
GithubTokenFailed,
ParallelBaseCheckpointFailed,
PullRequestFailed,
SandboxCleanupFailed,
SandboxGitUnavailable,
SandboxPreserved,
WorktreeSkippedNoGit,
}
#[must_use]
pub fn is_metadata_snapshot_compat_notice_code(code: &str) -> bool {
matches!(
code,
NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED | NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED
)
impl RunNoticeCode {
#[must_use]
pub fn is_metadata_snapshot_compat(self) -> bool {
matches!(
self,
Self::CheckpointMetadataWriteFailed | Self::CheckpointMetadataPushFailed
)
}
}
#[derive(

4
lib/crates/fabro-types/src/run_event/mod.rs
View file

@ -1372,7 +1372,7 @@ mod tests {
for body in [
EventBody::RunNotice(RunNoticeProps {
level: RunNoticeLevel::Warn,
code: "git_diff_failed".to_string(),
code: RunNoticeCode::GitDiffFailed.to_string(),
message: "git diff failed".to_string(),
exec_output_tail: Some(tail.clone()),
}),
@ -1408,7 +1408,7 @@ mod tests {
for body in [
EventBody::RunNotice(RunNoticeProps {
level: RunNoticeLevel::Warn,
code: "git_diff_failed".to_string(),
code: RunNoticeCode::GitDiffFailed.to_string(),
message: "git diff failed".to_string(),
exec_output_tail: None,
}),

2
lib/crates/fabro-types/src/run_event/stage.rs
View file

@ -57,6 +57,8 @@ pub struct StageFailedProps {
pub will_retry: bool,
#[serde(default)]
pub duration_ms: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub billing: Option<BilledModelUsage>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

177
lib/crates/fabro-types/src/run_projection.rs
View file

@ -4,9 +4,9 @@ use std::num::NonZeroU32;
use chrono::{DateTime, Utc};
use crate::{
Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro,
RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, StageCompletion, StageId,
StartRecord,
BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition,
PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord,
StageCompletion, StageId, StageState, StartRecord,
};
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
@ -61,6 +61,17 @@ pub struct StageProjection {
pub live_streaming: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub termination: Option<crate::CommandTermination>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<DateTime<Utc>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<u64>,
/// Server-internal billing usage for the latest attempt; not part of the
/// wire contract because `BilledModelUsage` is not modeled in OpenAPI.
/// Read only in-process by the billing handler.
#[serde(skip)]
pub usage: Option<BilledModelUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub state: Option<StageState>,
}
/// Convert a 1-based event sequence number into the `NonZeroU32` form used for
@ -78,6 +89,8 @@ impl StageProjection {
prompt: None,
response: None,
completion: None,
duration_ms: None,
usage: None,
provider_used: None,
diff: None,
script_invocation: None,
@ -90,8 +103,55 @@ impl StageProjection {
streams_separated: None,
live_streaming: None,
termination: None,
started_at: None,
state: None,
}
}
/// Effective lifecycle state derived from stored event data.
///
/// Falls back to deriving from `completion` for projections that predate
/// the stored `state` field, so old serialized projections still work
/// without a backfill.
#[must_use]
pub fn effective_state(&self) -> StageState {
self.state.unwrap_or_else(|| match &self.completion {
Some(completion) => StageState::from(completion.outcome),
None => StageState::Running,
})
}
/// Live wall-clock runtime in seconds.
///
/// While the stage is non-terminal (`Pending`, `Running`, or `Retrying`),
/// this returns the elapsed time since `started_at` so the UI can tick
/// client-side. Once terminal, the stored `duration_ms` is returned. This
/// also handles retries safely: a new `StageStarted` resets the state
/// back to `Running` and keeps the live computation correct even if a
/// previous attempt left a stale `duration_ms`.
#[must_use]
pub fn runtime_secs(&self, now: DateTime<Utc>) -> Option<f64> {
let state = self.effective_state();
if matches!(
state,
StageState::Running | StageState::Retrying | StageState::Pending
) {
return self.started_at.map(|started| {
now.signed_duration_since(started).num_milliseconds().max(0) as f64 / 1000.0
});
}
self.duration_ms.map(|ms| ms as f64 / 1000.0)
}
/// Begin a new attempt (or visit) for this stage: clear every
/// per-attempt field so prior-attempt data does not leak, then record
/// `started_at` and `state = Running`. Preserves `first_event_seq`
/// (identity / sort key).
pub fn begin_attempt(&mut self, started_at: DateTime<Utc>) {
*self = Self::new(self.first_event_seq);
self.started_at = Some(started_at);
self.state = Some(StageState::Running);
}
}
impl RunProjection {
@ -99,12 +159,32 @@ impl RunProjection {
self.stages.get(stage)
}
/// Iterate stages in `first_event_seq` order (the chronological order in
/// which each stage's first lifecycle event was recorded). Internal
/// storage is a `HashMap`, so iteration would otherwise be
/// non-deterministic; every caller wants chronological order, so we sort
/// here once instead of asking each caller to remember.
pub fn iter_stages(&self) -> impl Iterator<Item = (&StageId, &StageProjection)> {
self.stages.iter()
let mut entries: Vec<(&StageId, &StageProjection)> = self.stages.iter().collect();
entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| {
left_stage
.first_event_seq
.cmp(&right_stage.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
entries.into_iter()
}
/// Mutable counterpart of [`iter_stages`]. Same chronological ordering.
pub fn iter_stages_mut(&mut self) -> impl Iterator<Item = (&StageId, &mut StageProjection)> {
self.stages.iter_mut()
let mut entries: Vec<(&StageId, &mut StageProjection)> = self.stages.iter_mut().collect();
entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| {
left_stage
.first_event_seq
.cmp(&right_stage.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
entries.into_iter()
}
pub fn is_empty(&self) -> bool {
@ -186,3 +266,90 @@ impl RunProjection {
}
}
}
#[cfg(test)]
mod iter_stages_tests {
use std::num::NonZeroU32;
use super::RunProjection;
fn seq(n: u32) -> NonZeroU32 {
NonZeroU32::new(n).unwrap()
}
#[test]
fn iter_stages_yields_chronological_order_across_nodes() {
let mut p = RunProjection::default();
// Insert in non-monotonic seq order to exercise the sort.
p.stage_entry("c", 1, seq(30));
p.stage_entry("a", 1, seq(10));
p.stage_entry("b", 1, seq(20));
let order: Vec<&str> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.node_id())
.collect();
assert_eq!(order, vec!["a", "b", "c"]);
}
#[test]
fn iter_stages_orders_visits_within_a_node() {
let mut p = RunProjection::default();
// Visit 2 inserted first; visit 1's earlier first_event_seq must still
// win the chronological ordering.
p.stage_entry("verify", 2, seq(50));
p.stage_entry("verify", 1, seq(20));
let visits: Vec<u32> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.visit())
.collect();
assert_eq!(visits, vec![1, 2]);
}
#[test]
fn iter_stages_mut_yields_chronological_order() {
let mut p = RunProjection::default();
p.stage_entry("c", 1, seq(30));
p.stage_entry("a", 1, seq(10));
p.stage_entry("b", 1, seq(20));
let order: Vec<String> = p
.iter_stages_mut()
.map(|(stage_id, _)| stage_id.node_id().to_string())
.collect();
assert_eq!(order, vec!["a", "b", "c"]);
}
#[test]
fn iter_stages_tie_breaks_same_first_event_seq_by_stage_id() {
for _ in 0..128 {
let mut p = RunProjection::default();
p.stage_entry("verify", 2, seq(10));
p.stage_entry("build", 1, seq(10));
p.stage_entry("verify", 1, seq(10));
let order: Vec<String> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.to_string())
.collect();
assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]);
}
}
#[test]
fn iter_stages_mut_tie_breaks_same_first_event_seq_by_stage_id() {
for _ in 0..128 {
let mut p = RunProjection::default();
p.stage_entry("verify", 2, seq(10));
p.stage_entry("build", 1, seq(10));
p.stage_entry("verify", 1, seq(10));
let order: Vec<String> = p
.iter_stages_mut()
.map(|(stage_id, _)| stage_id.to_string())
.collect();
assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]);
}
}
}

39
lib/crates/fabro-types/src/stage_id.rs
View file

@ -1,4 +1,5 @@
use std::fmt;
use std::num::NonZeroU32;
use std::str::FromStr;
use serde::de::Error as _;
@ -7,16 +8,21 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct StageId {
node_id: String,
visit: u32,
visit: NonZeroU32,
}
impl StageId {
#[must_use]
pub fn new(node_id: impl Into<String>, visit: u32) -> Self {
Self {
Self::try_new(node_id, visit).expect("stage id visit must be greater than zero")
}
pub fn try_new(node_id: impl Into<String>, visit: u32) -> Result<Self, InvalidStageVisit> {
let visit = NonZeroU32::new(visit).ok_or(InvalidStageVisit)?;
Ok(Self {
node_id: node_id.into(),
visit,
}
})
}
#[must_use]
@ -26,7 +32,7 @@ impl StageId {
#[must_use]
pub fn visit(&self) -> u32 {
self.visit
self.visit.get()
}
}
@ -47,6 +53,17 @@ impl fmt::Display for ParseStageIdError {
impl std::error::Error for ParseStageIdError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InvalidStageVisit;
impl fmt::Display for InvalidStageVisit {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("stage id visit must be greater than zero")
}
}
impl std::error::Error for InvalidStageVisit {}
impl FromStr for StageId {
type Err = ParseStageIdError;
@ -67,7 +84,7 @@ impl FromStr for StageId {
let visit = visit
.parse()
.map_err(|err| ParseStageIdError(format!("invalid stage id visit: {err}")))?;
Ok(Self::new(node_id, visit))
Self::try_new(node_id, visit).map_err(|err| ParseStageIdError(err.to_string()))
}
}
@ -224,6 +241,18 @@ mod tests {
assert!(err.to_string().starts_with("invalid stage id visit:"));
}
#[test]
fn parse_rejects_zero_visit() {
let err = "code@0".parse::<StageId>().unwrap_err();
assert_eq!(err.to_string(), "stage id visit must be greater than zero");
}
#[test]
fn try_new_rejects_zero_visit() {
let err = StageId::try_new("code", 0).unwrap_err();
assert_eq!(err.to_string(), "stage id visit must be greater than zero");
}
#[test]
fn parse_rejects_empty_node_id() {
let err = "@3".parse::<StageId>().unwrap_err();

291
lib/crates/fabro-workflow/src/billing_rollup.rs Normal file
View file

@ -0,0 +1,291 @@
use std::collections::{BTreeMap, HashMap};
use fabro_types::{BilledModelUsage, BilledTokenCounts, RunProjection};
#[derive(Debug, Clone, PartialEq)]
pub struct ProjectionBillingStage {
pub node_id: String,
pub billing: BilledTokenCounts,
pub duration_ms: u64,
pub model_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProjectionBillingByModel {
pub model_id: String,
pub stages: i64,
pub billing: BilledTokenCounts,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct ProjectionBillingRollup {
pub stages: Vec<ProjectionBillingStage>,
pub totals: BilledTokenCounts,
pub by_model: Vec<ProjectionBillingByModel>,
pub runtime_ms: u64,
pub billed_visit_count: usize,
}
impl ProjectionBillingRollup {
#[must_use]
pub fn billing_if_present(&self) -> Option<BilledTokenCounts> {
(self.billed_visit_count > 0).then(|| self.totals.clone())
}
}
#[must_use]
pub fn billing_rollup_from_projection(projection: &RunProjection) -> ProjectionBillingRollup {
let mut stage_indices = HashMap::<String, usize>::new();
let mut stages = Vec::<ProjectionBillingStage>::new();
let mut by_model = BTreeMap::<String, ProjectionBillingByModel>::new();
let mut totals = BilledTokenCounts::default();
let mut runtime_ms = 0_u64;
let mut billed_visit_count = 0_usize;
for (stage_id, stage) in projection.iter_stages() {
if is_exit_stage(projection, stage_id.node_id()) {
continue;
}
if stage.completion.is_none() && stage.duration_ms.is_none() && stage.usage.is_none() {
continue;
}
let node_id = stage_id.node_id();
let index = *stage_indices.entry(node_id.to_string()).or_insert_with(|| {
let index = stages.len();
stages.push(ProjectionBillingStage {
node_id: node_id.to_string(),
billing: BilledTokenCounts::default(),
duration_ms: 0,
model_id: None,
});
index
});
let row = &mut stages[index];
if let Some(duration_ms) = stage.duration_ms {
row.duration_ms = row.duration_ms.saturating_add(duration_ms);
runtime_ms = runtime_ms.saturating_add(duration_ms);
}
if let Some(usage) = stage.usage.as_ref() {
billed_visit_count += 1;
row.model_id = Some(usage.model_id().to_string());
accumulate_usage(&mut row.billing, usage);
accumulate_usage(&mut totals, usage);
let model_id = usage.model_id().to_string();
let model_entry =
by_model
.entry(model_id.clone())
.or_insert_with(|| ProjectionBillingByModel {
model_id,
stages: 0,
billing: BilledTokenCounts::default(),
});
model_entry.stages += 1;
accumulate_usage(&mut model_entry.billing, usage);
}
}
ProjectionBillingRollup {
stages,
totals,
by_model: by_model.into_values().collect(),
runtime_ms,
billed_visit_count,
}
}
fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool {
projection
.spec()
.and_then(|spec| spec.graph().nodes.get(node_id))
.is_some_and(|node| node.handler_type() == Some("exit"))
}
fn accumulate_usage(counts: &mut BilledTokenCounts, usage: &BilledModelUsage) {
let tokens = usage.tokens();
counts.input_tokens += tokens.input_tokens;
counts.output_tokens += tokens.output_tokens;
counts.reasoning_tokens += tokens.reasoning_tokens;
counts.cache_read_tokens += tokens.cache_read_tokens;
counts.cache_write_tokens += tokens.cache_write_tokens;
counts.total_tokens += tokens.total_tokens();
if let Some(value) = usage.total_usd_micros {
*counts.total_usd_micros.get_or_insert(0) += value;
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use fabro_types::{
AttrValue, BilledModelUsage, Graph, Node, RunProjection, RunSpec, StageCompletion,
StageOutcome, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
use super::billing_rollup_from_projection;
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn rollup_groups_stage_rows_by_node_and_sums_retry_visit_usage() {
let mut projection = RunProjection::default();
let failed_usage = test_usage("gpt-old", 100, 10);
let success_usage = test_usage("gpt-new", 200, 20);
let first = projection.stage_entry("verify", 1, first_event_seq(1));
first.duration_ms = Some(1200);
first.usage = Some(failed_usage);
first.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: true,
},
notes: None,
failure_reason: Some("try again".to_string()),
timestamp: chrono::Utc::now(),
});
let second = projection.stage_entry("verify", 2, first_event_seq(2));
second.duration_ms = Some(800);
second.usage = Some(success_usage);
second.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "verify");
assert_eq!(rollup.stages[0].model_id.as_deref(), Some("gpt-new"));
assert_eq!(rollup.stages[0].duration_ms, 2000);
assert_eq!(rollup.stages[0].billing.input_tokens, 300);
assert_eq!(rollup.stages[0].billing.output_tokens, 30);
assert_eq!(rollup.stages[0].billing.total_usd_micros, Some(330));
assert_eq!(rollup.runtime_ms, 2000);
assert_eq!(rollup.totals.input_tokens, 300);
assert_eq!(rollup.totals.output_tokens, 30);
assert_eq!(rollup.totals.total_usd_micros, Some(330));
assert_eq!(rollup.billed_visit_count, 2);
assert_eq!(rollup.by_model.len(), 2);
assert_eq!(rollup.by_model[0].model_id, "gpt-new");
assert_eq!(rollup.by_model[0].stages, 1);
assert_eq!(rollup.by_model[0].billing.input_tokens, 200);
assert_eq!(rollup.by_model[1].model_id, "gpt-old");
assert_eq!(rollup.by_model[1].stages, 1);
assert_eq!(rollup.by_model[1].billing.input_tokens, 100);
}
#[test]
fn rollup_includes_completed_non_llm_stage_rows_with_zero_billing() {
let mut projection = RunProjection::default();
let stage = projection.stage_entry("start", 1, first_event_seq(1));
stage.duration_ms = Some(25);
stage.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "start");
assert_eq!(rollup.stages[0].duration_ms, 25);
assert!(rollup.stages[0].model_id.is_none());
assert_eq!(rollup.stages[0].billing.input_tokens, 0);
assert_eq!(rollup.runtime_ms, 25);
assert!(rollup.by_model.is_empty());
assert!(rollup.billing_if_present().is_none());
}
#[test]
fn rollup_excludes_terminal_exit_stage_rows() {
let mut projection = RunProjection::default();
projection.spec = Some(run_spec_with_exit_node());
let start = projection.stage_entry("start", 1, first_event_seq(1));
start.duration_ms = Some(25);
start.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let exit = projection.stage_entry("exit", 1, first_event_seq(2));
exit.duration_ms = Some(7);
exit.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "start");
assert_eq!(rollup.runtime_ms, 25);
}
fn run_spec_with_exit_node() -> RunSpec {
let mut graph = Graph::new("test");
graph.nodes.insert("start".to_string(), {
let mut node = Node::new("start");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
node
});
graph.nodes.insert("exit".to_string(), {
let mut node = Node::new("exit");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
node
});
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph,
workflow_slug: None,
source_directory: None,
labels: HashMap::new(),
provenance: None,
manifest_blob: None,
definition_blob: None,
git: None,
fork_source_ref: None,
in_place: false,
}
}
}

1
lib/crates/fabro-workflow/src/error.rs
View file

@ -1838,6 +1838,7 @@ mod tests {
failure: failure.clone(),
will_retry: false,
duration_ms: 0,
billing: None,
actor: None,
};

2
lib/crates/fabro-workflow/src/event.rs
View file

@ -8,7 +8,7 @@ mod stored_fields;
#[cfg(test)]
mod test_support;
pub use fabro_types::{EventBody, RunNoticeLevel};
pub use fabro_types::{EventBody, RunNoticeCode, RunNoticeLevel};
pub use self::convert::{to_run_event, to_run_event_at};
pub use self::emitter::Emitter;

35
lib/crates/fabro-workflow/src/event/convert.rs
View file

@ -290,12 +290,14 @@ fn event_body_from_event(event: &Event) -> EventBody {
failure,
will_retry,
duration_ms,
billing,
..
} => EventBody::StageFailed(fabro_types::StageFailedProps {
index: *index,
failure: Some(failure.clone()),
will_retry: *will_retry,
duration_ms: *duration_ms,
billing: billing.clone(),
}),
Event::StageRetrying {
index,
@ -1187,8 +1189,8 @@ mod tests {
use std::collections::BTreeMap;
use ::fabro_types::{
EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeLevel, RunProvenance,
StageId, SystemActorKind, fixtures, run_event as fabro_types,
EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, RunNoticeLevel,
RunProvenance, StageId, SystemActorKind, fixtures, run_event as fabro_types,
};
use chrono::Utc;
use fabro_agent::{AgentEvent, SandboxEvent};
@ -1198,7 +1200,7 @@ mod tests {
use crate::error::Error;
use crate::event::test_support::user_principal;
use crate::event::{Event, StageScope};
use crate::outcome::FailureDetail;
use crate::outcome::{BilledModelUsage, FailureDetail};
#[derive(Debug)]
struct EventTestCause;
@ -1220,6 +1222,28 @@ mod tests {
}
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(serde_json::json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn run_event_stage_completed_places_node_fields_in_header() {
let stored = to_run_event_at(
@ -1299,6 +1323,7 @@ mod tests {
#[test]
fn run_event_stage_failure_keeps_failure_detail() {
let usage = test_usage("gpt-5.2", 321, 54);
let stored = to_run_event(&fixtures::RUN_3, &Event::StageFailed {
node_id: "code".to_string(),
name: "Code".to_string(),
@ -1309,6 +1334,7 @@ mod tests {
),
will_retry: true,
duration_ms: 5000,
billing: Some(usage.clone()),
actor: None,
});
@ -1317,6 +1343,7 @@ mod tests {
assert_eq!(properties["failure"]["message"], "lint failed");
assert_eq!(properties["failure"]["failure_class"], "deterministic");
assert_eq!(properties["will_retry"], true);
assert_eq!(properties["billing"], serde_json::to_value(&usage).unwrap());
}
#[test]
@ -1636,7 +1663,7 @@ mod tests {
fn run_notice_maps_exec_output_tail_to_props() {
let stored = to_run_event(&fixtures::RUN_1, &Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "git_diff_failed".to_string(),
code: RunNoticeCode::GitDiffFailed.to_string(),
message: "git diff failed".to_string(),
exec_output_tail: Some(exec_tail()),
});

15
lib/crates/fabro-workflow/src/event/emitter.rs
View file

@ -1,7 +1,7 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicI64, Ordering};
use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeLevel};
use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeCode, RunNoticeLevel};
use chrono::Utc;
use fabro_agent::{WorktreeEvent, WorktreeEventCallback};
@ -76,15 +76,10 @@ impl Emitter {
self.emit_with_scope(event, Some(scope));
}
pub fn notice(
&self,
level: RunNoticeLevel,
code: impl Into<String>,
message: impl Into<String>,
) {
pub fn notice(&self, level: RunNoticeLevel, code: RunNoticeCode, message: impl Into<String>) {
self.emit(&Event::RunNotice {
level,
code: code.into(),
code: code.to_string(),
message: message.into(),
exec_output_tail: None,
});
@ -93,13 +88,13 @@ impl Emitter {
pub fn notice_with_tail(
&self,
level: RunNoticeLevel,
code: impl Into<String>,
code: RunNoticeCode,
message: impl Into<String>,
exec_output_tail: Option<ExecOutputTail>,
) {
self.emit(&Event::RunNotice {
level,
code: code.into(),
code: code.to_string(),
message: message.into(),
exec_output_tail,
});

1
lib/crates/fabro-workflow/src/event/events.rs
View file

@ -206,6 +206,7 @@ pub enum Event {
failure: FailureDetail,
will_retry: bool,
duration_ms: u64,
billing: Option<BilledModelUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
actor: Option<Principal>,
},

2
lib/crates/fabro-workflow/src/handler/agent.rs
View file

@ -54,6 +54,8 @@ pub trait CodergenBackend: Send + Sync {
_node: &Node,
_prompt: &str,
_system_prompt: Option<&str>,
_emitter: &Arc<Emitter>,
_stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
Err(Error::Validation(
"one_shot mode not supported by this backend".into(),

20
lib/crates/fabro-workflow/src/handler/llm/api.rs
View file

@ -379,6 +379,8 @@ impl CodergenBackend for AgentApiBackend {
node: &Node,
prompt: &str,
system_prompt: Option<&str>,
emitter: &Arc<Emitter>,
stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
let client = Client::from_source(self.source.as_ref())
.await
@ -452,14 +454,16 @@ impl CodergenBackend for AgentApiBackend {
let mut found = None;
for target in fallback_chain {
tracing::warn!(
stage = node.id.as_str(),
from_provider = from_provider.as_str(),
from_model = from_model.as_str(),
to_provider = target.provider.as_str(),
to_model = target.model.as_str(),
error = error_msg.as_str(),
"LLM provider failover (prompt)"
emitter.emit_scoped(
&Event::Failover {
stage: node.id.clone(),
from_provider: from_provider.clone(),
from_model: from_model.clone(),
to_provider: target.provider.clone(),
to_model: target.model.clone(),
error: error_msg.clone(),
},
stage_scope,
);
let max_tokens = node.max_tokens().or_else(|| {

6
lib/crates/fabro-workflow/src/handler/llm/cli.rs
View file

@ -901,9 +901,13 @@ impl CodergenBackend for BackendRouter {
node: &Node,
prompt: &str,
system_prompt: Option<&str>,
emitter: &Arc<Emitter>,
stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
// CLI backend doesn't support one_shot, always route to API
self.api_backend.one_shot(node, prompt, system_prompt).await
self.api_backend
.one_shot(node, prompt, system_prompt, emitter, stage_scope)
.await
}
}

8
lib/crates/fabro-workflow/src/handler/parallel.rs
View file

@ -12,7 +12,7 @@ use tokio::sync::Semaphore;
use super::{EngineServices, Handler};
use crate::context::{Context, WorkflowContext, keys};
use crate::error::Error;
use crate::event::{Event, StageScope};
use crate::event::{Event, RunNoticeCode, RunNoticeLevel, StageScope};
use crate::git::sanitize_ref_component;
use crate::hook_context::set_hook_node;
use crate::millis_u64;
@ -207,6 +207,12 @@ impl Handler for ParallelHandler {
error = %fabro_sandbox::display_for_log(&e),
"parallel base checkpoint failed"
);
services.run.emitter.notice_with_tail(
RunNoticeLevel::Warn,
RunNoticeCode::ParallelBaseCheckpointFailed,
format!("Could not checkpoint base state before parallel branches: {e}"),
fabro_sandbox::default_redacted_output_tail(&e),
);
None
}
}

23
lib/crates/fabro-workflow/src/handler/prompt.rs
View file

@ -113,7 +113,13 @@ impl Handler for PromptHandler {
let (response_text, stage_usage, backend_files_touched) =
if let Some(backend) = &self.backend {
let result = backend
.one_shot(node, &prompt, system_prompt.as_deref())
.one_shot(
node,
&prompt,
system_prompt.as_deref(),
&services.run.emitter,
&stage_scope,
)
.await;
match result {
Ok(CodergenResult::Full(outcome)) => return Ok(outcome),
@ -197,6 +203,7 @@ mod tests {
use tokio_util::sync::CancellationToken;
use super::*;
use crate::event::Emitter;
fn make_services() -> EngineServices {
EngineServices::test_default()
@ -221,7 +228,7 @@ mod tests {
let mut services = EngineServices::test_default();
services.run = services
.run
.with_emitter(Arc::new(crate::event::Emitter::new(fixtures::RUN_1)))
.with_emitter(Arc::new(Emitter::new(fixtures::RUN_1)))
.with_run_store(run_store.clone().into());
let logger = crate::event::StoreProgressLogger::new(run_store.clone());
logger.register(services.run.emitter.as_ref());
@ -277,7 +284,7 @@ mod tests {
_prompt: &str,
_context: &Context,
_thread_id: Option<&str>,
_emitter: &Arc<crate::event::Emitter>,
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
@ -290,6 +297,8 @@ mod tests {
_node: &Node,
_prompt: &str,
_system_prompt: Option<&str>,
_emitter: &Arc<Emitter>,
_stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "one-shot response".to_string(),
@ -338,7 +347,7 @@ mod tests {
_prompt: &str,
_context: &Context,
_thread_id: Option<&str>,
_emitter: &Arc<crate::event::Emitter>,
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
@ -351,6 +360,8 @@ mod tests {
_node: &Node,
_prompt: &str,
_system_prompt: Option<&str>,
_emitter: &Arc<Emitter>,
_stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "one-shot response".to_string(),
@ -396,7 +407,7 @@ mod tests {
_prompt: &str,
_context: &Context,
_thread_id: Option<&str>,
_emitter: &Arc<crate::event::Emitter>,
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn fabro_agent::Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
@ -409,6 +420,8 @@ mod tests {
_node: &Node,
prompt: &str,
system_prompt: Option<&str>,
_emitter: &Arc<Emitter>,
_stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(prompt.to_string());
*self.captured_system_prompt.lock().unwrap() = Some(system_prompt.map(String::from));

189
lib/crates/fabro-workflow/src/lib.rs
View file

@ -20,6 +20,7 @@ use std::sync::Arc;
use fabro_retro::retro::CompletedStage;
use fabro_store::EventEnvelope;
use fabro_types::{EventBody, StageId};
/// Callback invoked when a workflow node starts executing.
pub type OnNodeCallback = Option<Arc<dyn Fn(&str) + Send + Sync>>;
@ -86,34 +87,190 @@ pub fn build_completed_stages(cp: &records::Checkpoint, run_failed: bool) -> Vec
stages
}
pub fn extract_stage_durations_from_events(events: &[EventEnvelope]) -> HashMap<String, u64> {
/// Extract the `duration_ms` from a `stage.completed` / `stage.failed`
/// event body, or `None` for any other variant.
fn stage_completion_duration_ms(body: &EventBody) -> Option<u64> {
match body {
EventBody::StageCompleted(props) => Some(props.duration_ms),
EventBody::StageFailed(props) => Some(props.duration_ms),
_ => None,
}
}
/// Extract per-stage (node_id, visit) durations from `stage.completed` /
/// `stage.failed` events. Keys on the full [`StageId`] so multi-visit stages
/// (e.g. a looped `verify` node) keep distinct durations.
///
/// This is the canonical primitive; [`total_stage_duration_by_node`] and
/// [`latest_stage_duration_by_node`] are explicit rollups built on top of it.
pub fn extract_stage_durations_by_stage_id(events: &[EventEnvelope]) -> HashMap<StageId, u64> {
let mut durations = HashMap::new();
for envelope in events {
let event = &envelope.event;
let event_name = event.event_name();
if event_name != "stage.completed" && event_name != "stage.failed" {
continue;
}
let Some(node_id) = event.node_id.as_deref() else {
let Some(duration_ms) = stage_completion_duration_ms(&envelope.event.body) else {
continue;
};
let Some(duration_ms) = event
.properties()
.ok()
.and_then(|properties| properties.get("duration_ms").cloned())
.and_then(|duration| duration.as_u64())
else {
let Some(stage_id) = envelope.event.stage_id.as_ref() else {
continue;
};
durations.insert(node_id.to_string(), duration_ms);
durations.insert(stage_id.clone(), duration_ms);
}
durations
}
/// Total duration spent in each node, summed across every visit. Use for
/// billing/usage where a retried node should count its full time.
pub fn total_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap<String, u64> {
let mut totals: HashMap<String, u64> = HashMap::new();
for (stage_id, duration_ms) in extract_stage_durations_by_stage_id(events) {
*totals.entry(stage_id.node_id().to_string()).or_default() += duration_ms;
}
totals
}
/// Duration of each node's most recent visit (the highest visit number). Use
/// for run summaries and retros where the table shows one row per node and
/// "the last attempt" is the right representative.
pub fn latest_stage_duration_by_node(events: &[EventEnvelope]) -> HashMap<String, u64> {
let mut entries: Vec<(StageId, u64)> = extract_stage_durations_by_stage_id(events)
.into_iter()
.collect();
entries.sort_by_key(|(stage_id, _)| stage_id.visit());
let mut latest = HashMap::new();
for (stage_id, duration_ms) in entries {
latest.insert(stage_id.node_id().to_string(), duration_ms);
}
latest
}
#[cfg(test)]
mod duration_tests {
use chrono::{TimeZone, Utc};
use fabro_store::EventEnvelope;
use fabro_types::run_event::{StageCompletedProps, StageFailedProps};
use fabro_types::{EventBody, RunEvent, StageId, StageOutcome, fixtures};
use super::{
extract_stage_durations_by_stage_id, latest_stage_duration_by_node,
total_stage_duration_by_node,
};
fn completed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope {
let event = RunEvent {
id: format!("evt_{seq}"),
ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
run_id: fixtures::RUN_1,
node_id: Some(node.to_string()),
node_label: None,
stage_id: Some(StageId::new(node, visit)),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body: EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: vec![],
billing: None,
failure: None,
notes: None,
files_touched: vec![],
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
}),
};
EventEnvelope { seq, event }
}
fn failed_event(seq: u32, node: &str, visit: u32, duration_ms: u64) -> EventEnvelope {
let event = RunEvent {
id: format!("evt_{seq}"),
ts: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
run_id: fixtures::RUN_1,
node_id: Some(node.to_string()),
node_label: None,
stage_id: Some(StageId::new(node, visit)),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body: EventBody::StageFailed(StageFailedProps {
index: 0,
failure: None,
will_retry: true,
duration_ms,
billing: None,
}),
};
EventEnvelope { seq, event }
}
#[test]
fn extract_keys_durations_by_full_stage_id() {
let events = vec![
completed_event(1, "verify", 1, 100),
completed_event(2, "verify", 2, 200),
];
let durations = extract_stage_durations_by_stage_id(&events);
assert_eq!(
durations.get(&StageId::new("verify", 1)).copied(),
Some(100)
);
assert_eq!(
durations.get(&StageId::new("verify", 2)).copied(),
Some(200)
);
}
#[test]
fn total_sums_across_visits_per_node() {
let events = vec![
completed_event(1, "verify", 1, 100),
completed_event(2, "verify", 2, 200),
completed_event(3, "build", 1, 50),
];
let totals = total_stage_duration_by_node(&events);
assert_eq!(totals.get("verify").copied(), Some(300));
assert_eq!(totals.get("build").copied(), Some(50));
}
#[test]
fn latest_picks_highest_visit_regardless_of_input_order() {
// Visit 2 appears in the events vector before visit 1; the result
// must still reflect visit 2's duration (the latest visit).
let events = vec![
completed_event(1, "verify", 2, 999),
completed_event(2, "verify", 1, 100),
];
let latest = latest_stage_duration_by_node(&events);
assert_eq!(latest.get("verify").copied(), Some(999));
}
#[test]
fn stage_failed_durations_are_included() {
let events = vec![failed_event(1, "verify", 1, 75)];
let durations = extract_stage_durations_by_stage_id(&events);
assert_eq!(durations.get(&StageId::new("verify", 1)).copied(), Some(75));
}
}
#[doc(hidden)]
pub mod artifact;
pub mod artifact_snapshot;
pub mod artifact_upload;
pub mod billing_rollup;
pub mod command_log;
pub(crate) mod condition;
pub mod context;
@ -142,6 +299,10 @@ pub mod run_control;
pub(crate) mod run_dir;
pub mod run_lookup;
pub use billing_rollup::{
ProjectionBillingByModel, ProjectionBillingRollup, ProjectionBillingStage,
billing_rollup_from_projection,
};
pub use error::{Error, FailureCategory, FailureSignature, FailureSignatureExt, Result};
pub use manifest_path::ManifestPath;
pub mod run_materialization;

10
lib/crates/fabro-workflow/src/lifecycle/artifact.rs
View file

@ -16,7 +16,7 @@ use tokio::time::sleep;
use crate::artifact::{normalize_durable_updates, offload_large_values, sync_artifacts_to_env};
use crate::artifact_snapshot::collect_artifacts;
use crate::artifact_upload::ArtifactSink;
use crate::event::{Emitter, Event, RunNoticeLevel};
use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::lifecycle::event::{stage_scope_for, stage_visit};
use crate::outcome::BilledModelUsage;
@ -125,7 +125,7 @@ impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
{
self.emitter.notice(
RunNoticeLevel::Warn,
"artifact_upload_failed",
RunNoticeCode::ArtifactUploadFailed,
format!("[node: {node_id}] artifact upload failed: {err}"),
);
return Ok(());
@ -151,7 +151,7 @@ impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
Err(e) => {
self.emitter.notice(
RunNoticeLevel::Warn,
"artifact_collection_failed",
RunNoticeCode::ArtifactCollectionFailed,
format!("[node: {node_id}] artifact collection failed: {e}"),
);
}
@ -174,7 +174,7 @@ impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
{
self.emitter.notice(
RunNoticeLevel::Warn,
"artifact_offload_failed",
RunNoticeCode::ArtifactOffloadFailed,
format!("[node: {node_id}] artifact offload failed: {e}"),
);
}
@ -187,7 +187,7 @@ impl RunLifecycle<WorkflowGraph> for ArtifactLifecycle {
{
self.emitter.notice(
RunNoticeLevel::Warn,
"artifact_sync_failed",
RunNoticeCode::ArtifactSyncFailed,
format!("[node: {node_id}] artifact sync failed: {e}"),
);
}

2
lib/crates/fabro-workflow/src/lifecycle/event.rs
View file

@ -224,6 +224,7 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
failure,
will_retry: true,
duration_ms,
billing: outcome.usage.clone(),
actor,
},
&scope,
@ -275,6 +276,7 @@ impl RunLifecycle<WorkflowGraph> for EventLifecycle {
failure,
will_retry: false,
duration_ms,
billing: outcome.usage.clone(),
actor,
},
&scope,

43
lib/crates/fabro-workflow/src/lifecycle/git.rs
View file

@ -14,7 +14,7 @@ use fabro_util::error::collect_causes;
use fabro_util::time::elapsed_ms;
use crate::artifact;
use crate::event::{Emitter, Event, RunNoticeLevel, StageScope};
use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel, StageScope};
use crate::graph::{WorkflowGraph, WorkflowNode};
use crate::lifecycle::event::stage_scope_for;
use crate::outcome::BilledModelUsage;
@ -128,7 +128,10 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
None,
None,
);
self.emit_metadata_warning("checkpoint_metadata_write_failed", message);
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
}
},
Err(err) => {
@ -145,7 +148,10 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
None,
None,
);
self.emit_metadata_warning("checkpoint_metadata_write_failed", message);
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
}
}
}
@ -217,7 +223,7 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
Some(&scope),
);
self.emit_metadata_warning(
"checkpoint_metadata_write_failed",
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
None
@ -239,7 +245,10 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
None,
Some(&scope),
);
self.emit_metadata_warning("checkpoint_metadata_write_failed", message);
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
None
}
}
@ -292,6 +301,12 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
error = %fabro_sandbox::display_for_log(&err),
"git push from run lifecycle failed"
);
self.emitter.notice_with_tail(
RunNoticeLevel::Warn,
RunNoticeCode::GitPushFailed,
format!("Failed to push run branch {branch}: {err}"),
exec_output_tail.clone(),
);
(false, exec_output_tail)
}
};
@ -321,7 +336,7 @@ impl RunLifecycle<WorkflowGraph> for GitLifecycle {
fabro_sandbox::default_redacted_output_tail(&err);
self.emitter.notice_with_tail(
RunNoticeLevel::Warn,
"git_diff_failed",
RunNoticeCode::GitDiffFailed,
format!("[node: {node_id}] git diff failed: {err}"),
exec_output_tail,
);
@ -395,7 +410,10 @@ impl GitLifecycle {
Some(snapshot.bytes),
scope,
);
self.emit_metadata_warning("checkpoint_metadata_push_failed", message);
self.emit_metadata_warning(
RunNoticeCode::CheckpointMetadataPushFailed,
message,
);
} else {
self.emit_metadata_snapshot_completed(
phase,
@ -421,7 +439,7 @@ impl GitLifecycle {
None,
scope,
);
self.emit_metadata_warning("checkpoint_metadata_write_failed", message);
self.emit_metadata_warning(RunNoticeCode::CheckpointMetadataWriteFailed, message);
None
}
}
@ -506,14 +524,9 @@ impl GitLifecycle {
}
}
fn emit_metadata_warning(&self, code: &str, message: String) {
fn emit_metadata_warning(&self, code: RunNoticeCode, message: String) {
if self.metadata_runtime.mark_metadata_degraded() {
self.emitter.emit(&Event::RunNotice {
level: RunNoticeLevel::Warn,
code: code.to_string(),
message,
exec_output_tail: None,
});
self.emitter.notice(RunNoticeLevel::Warn, code, message);
}
}
}

179
lib/crates/fabro-workflow/src/pipeline/finalize.rs
View file

@ -10,7 +10,7 @@ use fabro_util::time::elapsed_ms;
use super::types::{Concluded, FinalizeOptions, Retroed};
use crate::error::Error;
use crate::event::{Event, RunNoticeLevel};
use crate::event::{Event, RunNoticeCode, RunNoticeLevel};
use crate::outcome::{Outcome, OutcomeExt, StageOutcome};
use crate::records::{Checkpoint, Conclusion, StageSummary};
use crate::run_metadata::MetadataSnapshot;
@ -19,6 +19,7 @@ use crate::run_status::{FailureReason, RunStatus, SuccessReason};
use crate::runtime_store::RunStoreHandle;
use crate::sandbox_git::git_diff_with_timeout;
use crate::services::RunServices;
use crate::{ProjectionBillingRollup, billing_rollup_from_projection};
pub fn classify_engine_result(
engine_result: &Result<Outcome, Error>,
@ -68,22 +69,22 @@ pub(crate) async fn build_conclusion_from_store(
run_duration_ms: u64,
final_git_commit_sha: Option<String>,
) -> Conclusion {
let (state_result, events_result) = tokio::join!(run_store.state(), run_store.list_events());
let projection = state_result.ok();
let projection = run_store.state().await.ok();
let projection_order = projection
.as_ref()
.map(stage_projection_order)
.unwrap_or_default();
let projection_billing = projection
.as_ref()
.map(billing_rollup_from_projection)
.unwrap_or_default();
let checkpoint = projection
.as_ref()
.and_then(|state| state.checkpoint.as_ref());
let stage_durations = events_result
.map(|events| crate::extract_stage_durations_from_events(&events))
.unwrap_or_default();
build_conclusion_from_parts(
checkpoint,
&stage_durations,
&projection_billing,
&projection_order,
status,
failure_reason,
@ -94,7 +95,7 @@ pub(crate) async fn build_conclusion_from_store(
fn build_conclusion_from_parts(
checkpoint: Option<&Checkpoint>,
stage_durations: &HashMap<String, u64>,
projection_billing: &ProjectionBillingRollup,
projection_order: &HashMap<String, u32>,
status: StageOutcome,
failure_reason: Option<String>,
@ -105,6 +106,11 @@ fn build_conclusion_from_parts(
// while the other checkpoint maps are keyed by node_id. Dedupe to one row
// per node so the stages table matches the deduped billing total.
let (stages, total_retries) = if let Some(cp) = checkpoint {
let billing_by_node = projection_billing
.stages
.iter()
.map(|stage| (stage.node_id.as_str(), stage))
.collect::<HashMap<_, _>>();
let mut stage_rows = Vec::new();
let mut seen = std::collections::HashSet::new();
let mut retries_sum: u32 = 0;
@ -130,7 +136,6 @@ fn build_conclusion_from_parts(
}
for (original_checkpoint_order, node_id) in stage_order {
let outcome = cp.node_outcomes.get(node_id);
let retries = cp
.node_retries
.get(node_id)
@ -138,14 +143,13 @@ fn build_conclusion_from_parts(
.unwrap_or(1)
.saturating_sub(1);
retries_sum += retries;
let billing = billing_by_node.get(node_id);
let summary = StageSummary {
stage_id: node_id.to_string(),
stage_label: node_id.to_string(),
duration_ms: stage_durations.get(node_id).copied().unwrap_or(0),
billing_usd_micros: outcome
.and_then(|o| o.usage.as_ref())
.and_then(|usage| usage.total_usd_micros),
duration_ms: billing.map_or(0, |stage| stage.duration_ms),
billing_usd_micros: billing.and_then(|stage| stage.billing.total_usd_micros),
retries,
};
stage_rows.push((
@ -176,7 +180,7 @@ fn build_conclusion_from_parts(
failure_reason,
final_git_commit_sha,
stages,
billing: checkpoint.and_then(billing_from_checkpoint),
billing: projection_billing.billing_if_present(),
total_retries,
}
}
@ -235,7 +239,11 @@ pub async fn write_finalize_commit(
None,
None,
);
emit_metadata_warning(services, "checkpoint_metadata_write_failed", message);
emit_metadata_warning(
services,
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
return;
}
};
@ -256,7 +264,11 @@ pub async fn write_finalize_commit(
None,
None,
);
emit_metadata_warning(services, "checkpoint_metadata_write_failed", message);
emit_metadata_warning(
services,
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
return;
}
};
@ -277,7 +289,11 @@ pub async fn write_finalize_commit(
Some(snapshot.entry_count),
Some(snapshot.bytes),
);
emit_metadata_warning(services, "checkpoint_metadata_push_failed", message);
emit_metadata_warning(
services,
RunNoticeCode::CheckpointMetadataPushFailed,
message,
);
} else {
emit_metadata_snapshot_completed(services, phase, meta_branch, started, &snapshot);
}
@ -296,7 +312,11 @@ pub async fn write_finalize_commit(
None,
None,
);
emit_metadata_warning(services, "checkpoint_metadata_write_failed", message);
emit_metadata_warning(
services,
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
}
}
}
@ -359,7 +379,7 @@ fn emit_metadata_snapshot_failed(
});
}
fn emit_metadata_warning(services: &RunServices, code: &str, message: String) {
fn emit_metadata_warning(services: &RunServices, code: RunNoticeCode, message: String) {
if services.metadata_runtime.mark_metadata_degraded() {
services.emitter.notice(RunNoticeLevel::Warn, code, message);
}
@ -383,7 +403,7 @@ async fn compute_final_patch(
Err(err) => {
services.emitter.notice(
RunNoticeLevel::Warn,
"git_diff_failed",
RunNoticeCode::GitDiffFailed,
format!("final diff failed: {err}"),
);
None
@ -391,15 +411,8 @@ async fn compute_final_patch(
}
}
/// Iterates `node_outcomes.values()` rather than `completed_nodes` to avoid
/// over-counting the last visit's usage on looping workflows.
pub(crate) fn billing_from_checkpoint(cp: &Checkpoint) -> Option<BilledTokenCounts> {
let usage: Vec<_> = cp
.node_outcomes
.values()
.filter_map(|o| o.usage.clone())
.collect();
(!usage.is_empty()).then(|| BilledTokenCounts::from_billed_usage(&usage))
pub(crate) fn billing_from_projection(projection: &RunProjection) -> Option<BilledTokenCounts> {
billing_rollup_from_projection(projection).billing_if_present()
}
pub(crate) fn build_terminal_event(
@ -503,7 +516,6 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result<Con
let (final_status, failure_reason, _run_status) = classify_engine_result(&outcome);
let events = services.run_store.list_events().await.unwrap_or_default();
let stage_durations = crate::extract_stage_durations_from_events(&events);
let artifact_count = events
.iter()
.filter(|envelope| matches!(envelope.event.body, EventBody::ArtifactCaptured(_)))
@ -513,12 +525,16 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result<Con
.as_ref()
.map(stage_projection_order)
.unwrap_or_default();
let projection_billing = projection
.as_ref()
.map(billing_rollup_from_projection)
.unwrap_or_default();
let checkpoint = projection
.as_ref()
.and_then(|state| state.checkpoint.as_ref());
let conclusion = build_conclusion_from_parts(
checkpoint,
&stage_durations,
&projection_billing,
&projection_order,
final_status,
failure_reason,
@ -534,7 +550,7 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result<Con
if services.metadata_runtime.metadata_degraded() {
services.emitter.notice(
RunNoticeLevel::Warn,
"checkpoint_metadata_degraded",
RunNoticeCode::CheckpointMetadataDegraded,
"checkpoint metadata archive writes were degraded for this run".to_string(),
);
}
@ -556,9 +572,11 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result<Con
} else {
format!("sandbox preserved: {info}")
};
services
.emitter
.notice(RunNoticeLevel::Info, "sandbox_preserved", message);
services.emitter.notice(
RunNoticeLevel::Info,
RunNoticeCode::SandboxPreserved,
message,
);
}
if let Err(e) = cleanup_sandbox(
&services,
@ -572,7 +590,7 @@ pub async fn finalize(retroed: Retroed, options: &FinalizeOptions) -> Result<Con
let exec_output_tail = fabro_sandbox::default_redacted_output_tail(&e);
services.emitter.notice_with_tail(
RunNoticeLevel::Warn,
"sandbox_cleanup_failed",
RunNoticeCode::SandboxCleanupFailed,
format!("sandbox cleanup failed: {}", e.display_with_causes()),
exec_output_tail,
);
@ -601,7 +619,8 @@ mod tests {
use fabro_store::{Database, EventEnvelope, RunDatabase, RunProjection};
use fabro_types::run_event::{MetadataSnapshotFailureKind, MetadataSnapshotPhase};
use fabro_types::{
EventBody, RunBlobId, RunEvent, RunId, WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, EventBody, RunBlobId, RunEvent, RunId, StageCompletion, WorkflowSettings,
first_event_seq, fixtures,
};
use object_store::memory::InMemory;
@ -737,6 +756,28 @@ mod tests {
}
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(serde_json::json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn conclusion_stage_order_follows_projection_first_event_order() {
let mut projection = RunProjection::default();
@ -753,7 +794,7 @@ mod tests {
let conclusion = build_conclusion_from_parts(
Some(&checkpoint),
&HashMap::new(),
&ProjectionBillingRollup::default(),
&projection_order,
StageOutcome::Succeeded,
None,
@ -788,7 +829,7 @@ mod tests {
let conclusion = build_conclusion_from_parts(
Some(&checkpoint),
&HashMap::new(),
&ProjectionBillingRollup::default(),
&projection_order,
StageOutcome::Succeeded,
None,
@ -804,6 +845,66 @@ mod tests {
assert_eq!(stage_ids, vec!["skipped", "finished"]);
}
#[test]
fn conclusion_billing_sums_retry_visit_usage_from_projection() {
let mut projection = RunProjection::default();
let failed_usage = test_usage("gpt-old", 100, 10);
let success_usage = test_usage("gpt-new", 200, 20);
let failed = projection.stage_entry("verify", 1, first_event_seq(1));
failed.duration_ms = Some(1200);
failed.usage = Some(failed_usage);
failed.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: true,
},
notes: None,
failure_reason: Some("try again".to_string()),
timestamp: chrono::Utc::now(),
});
let succeeded = projection.stage_entry("verify", 2, first_event_seq(2));
succeeded.duration_ms = Some(800);
succeeded.usage = Some(success_usage.clone());
succeeded.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let projection_order = stage_projection_order(&projection);
let projection_billing = billing_rollup_from_projection(&projection);
let mut latest_outcome = Outcome::success();
latest_outcome.usage = Some(success_usage);
latest_outcome.duration_ms = Some(800);
let mut checkpoint = checkpoint_with(
vec!["verify", "verify"],
HashMap::from([("verify".to_string(), latest_outcome)]),
);
checkpoint.node_retries.insert("verify".to_string(), 2);
let conclusion = build_conclusion_from_parts(
Some(&checkpoint),
&projection_billing,
&projection_order,
StageOutcome::Succeeded,
None,
10,
None,
);
assert_eq!(conclusion.billing.as_ref().unwrap().input_tokens, 300);
assert_eq!(conclusion.billing.as_ref().unwrap().output_tokens, 30);
assert_eq!(
conclusion.billing.as_ref().unwrap().total_usd_micros,
Some(330)
);
assert_eq!(conclusion.stages.len(), 1);
assert_eq!(conclusion.stages[0].stage_id, "verify");
assert_eq!(conclusion.stages[0].duration_ms, 2000);
assert_eq!(conclusion.stages[0].billing_usd_micros, Some(330));
assert_eq!(conclusion.stages[0].retries, 1);
}
fn test_services(
run_store: RunStoreHandle,
emitter: Arc<Emitter>,

59
lib/crates/fabro-workflow/src/pipeline/initialize.rs
View file

@ -27,7 +27,7 @@ use tokio::time::timeout as tokio_timeout;
use super::types::{InitOptions, Initialized, LlmSpec, Persisted, SandboxEnvSpec};
use crate::devcontainer_bridge::{devcontainer_to_snapshot_config, run_devcontainer_lifecycle};
use crate::error::Error;
use crate::event::{Emitter, Event, RunNoticeLevel};
use crate::event::{Emitter, Event, RunNoticeCode, RunNoticeLevel};
use crate::git::RUN_BRANCH_PREFIX;
use crate::handler::llm::{AgentApiBackend, AgentCliBackend, BackendRouter};
use crate::handler::{HandlerRegistry, default_registry};
@ -155,7 +155,7 @@ fn resolve_worktree_plan(options: &mut InitOptions) -> Option<WorktreePlan> {
if let Some(env_name) = env_name {
options.emitter.notice(
RunNoticeLevel::Warn,
"dirty_worktree",
RunNoticeCode::DirtyWorktree,
format!("Uncommitted changes will not be included in the {env_name}."),
);
}
@ -200,6 +200,14 @@ fn resolve_worktree_plan(options: &mut InitOptions) -> Option<WorktreePlan> {
})
}
fn worktree_skipped_notice(mode: Option<WorktreeMode>) -> Option<(RunNoticeCode, &'static str)> {
matches!(mode, Some(WorktreeMode::Always)).then_some((
RunNoticeCode::WorktreeSkippedNoGit,
"Worktree mode `always` requested but no Git repository was found; running without a \
worktree.",
))
}
fn git_setup_intent(run_options: &RunOptions) -> GitSetupIntent {
if let Some(source) = run_options.fork_source_ref.as_ref() {
GitSetupIntent::ForkFromCheckpoint {
@ -239,11 +247,14 @@ async fn build_sandbox_env(
Ok(token) => {
env.insert("GITHUB_TOKEN".to_string(), token);
}
Err(e) => emitter.notice(
RunNoticeLevel::Warn,
"github_token_failed",
format!("Failed to mint GitHub token: {e}"),
),
Err(e) => {
tracing::warn!(error = %e, "Failed to mint GitHub token");
emitter.notice(
RunNoticeLevel::Warn,
RunNoticeCode::GithubTokenFailed,
format!("Failed to mint GitHub token: {e}"),
);
}
}
}
}
@ -516,6 +527,13 @@ pub async fn initialize(
))
};
if worktree_plan.is_some() && !worktree_created {
if let Some((code, message)) = worktree_skipped_notice(options.worktree_mode) {
tracing::warn!(
worktree_mode = ?options.worktree_mode,
"worktree skipped: cwd is not a git repository"
);
options.emitter.notice(RunNoticeLevel::Warn, code, message);
}
options.run_options.git = None;
}
let cleanup_guard = scopeguard::guard(Arc::clone(&sandbox), |sandbox| {
@ -593,7 +611,8 @@ pub async fn initialize(
.is_some();
if !has_run_branch {
let intent = git_setup_intent(&options.run_options);
if sandbox.origin_url().is_some() {
let sandbox_has_origin = sandbox.origin_url().is_some();
if sandbox_has_origin {
sandbox_git
.ensure_git_available(&*sandbox)
.await
@ -619,7 +638,16 @@ pub async fn initialize(
options.run_options.base_branch = info.base_branch;
}
}
Ok(None) => {}
Ok(None) => {
if sandbox_has_origin {
options.emitter.notice(
RunNoticeLevel::Warn,
RunNoticeCode::SandboxGitUnavailable,
"Sandbox could not set up Git despite a configured origin; running \
without checkpointing or PR support.",
);
}
}
Err(e) => {
return Err(Error::engine_with_source("Sandbox git setup failed", &e));
}
@ -700,7 +728,7 @@ pub async fn initialize(
if metadata_runtime.mark_metadata_degraded() {
options.emitter.notice(
RunNoticeLevel::Warn,
"checkpoint_metadata_write_failed",
RunNoticeCode::CheckpointMetadataWriteFailed,
message,
);
}
@ -1008,6 +1036,17 @@ mod tests {
assert!(options.run_options.git.is_none());
}
#[test]
fn worktree_skipped_notice_only_warns_for_always() {
assert!(worktree_skipped_notice(None).is_none());
assert!(worktree_skipped_notice(Some(WorktreeMode::Clean)).is_none());
assert!(worktree_skipped_notice(Some(WorktreeMode::Dirty)).is_none());
assert!(worktree_skipped_notice(Some(WorktreeMode::Never)).is_none());
let (code, _) = worktree_skipped_notice(Some(WorktreeMode::Always)).unwrap();
assert_eq!(code, RunNoticeCode::WorktreeSkippedNoGit);
}
#[tokio::test]
async fn initialize_prepares_sandbox_and_uses_persisted_run_dir() {
let temp = tempfile::tempdir().unwrap();

5
lib/crates/fabro-workflow/src/pipeline/mod.rs
View file

@ -12,14 +12,15 @@ mod validate;
pub use execute::execute;
pub use fabro_types::PullRequestRecord;
pub(crate) use finalize::{
billing_from_checkpoint, build_conclusion_from_store, build_terminal_event,
billing_from_projection, build_conclusion_from_store, build_terminal_event,
};
pub use finalize::{classify_engine_result, finalize, write_finalize_commit};
pub use initialize::initialize;
pub use parse::parse;
pub(crate) use persist::persist;
pub use pull_request::{
AutoMergeOptions, OpenPullRequestRequest, build_pr_body, maybe_open_pull_request, pull_request,
AutoMergeOptions, OpenPullRequestRequest, PrContent, build_pr_content, maybe_open_pull_request,
pull_request,
};
pub use retro::{retro, run_retro};
pub use transform::transform;

54
lib/crates/fabro-workflow/src/pipeline/prompts/pr_body.md Normal file
View file

@ -0,0 +1,54 @@
You are writing a pull request title and description for a code change produced by an AI workflow.
OUTPUT FORMAT
Return a JSON object with exactly two fields:
- "title": a one-line title, max 72 characters, no trailing period.
- "body": the markdown body as described below.
DO NOT INCLUDE in the body
- A `#` or `##` title heading at the top -- the title goes in the `title` field.
- A "Retro" section, "Fabro Details" section, cost/duration table, or "Generated with" footer -- those are appended programmatically after your output.
- The full plan text -- the full plan is appended programmatically as a <details> block.
- Bare `#1`, `#2` list prefixes -- GitHub auto-links those as issue references. Use plain `1.`, `2.` instead.
- A test plan unless the testing approach is non-obvious.
SIZE THE BODY TO THE CHANGE
First classify along two axes from the diff:
- Size: how many files changed, how large the diff is.
- Complexity: trivial (rename / typo / dep bump / config) vs. design decisions / new patterns / cross-cutting concerns.
Then write at the matching depth:
| Profile | Body shape |
|---|---|
| Small + simple (typo, config, dep bump) | 1-2 sentences, no headers, total under ~300 characters |
| Small + non-trivial (targeted bugfix, behavioral change) | Short "Problem / Fix" narrative, 3-5 sentences. No headers unless two distinct concerns. |
| Medium feature or refactor | Summary paragraph, then a section explaining what changed and why. Call out design decisions. |
| Large or architecturally significant | Full narrative: problem context, approach chosen (and why), key decisions, migration/rollback notes if relevant. |
| Performance improvement | Include before/after measurements if available. A markdown table works well here. |
Brevity matters for small changes. A 3-line bugfix with a 20-line description signals miscalibration. When in doubt, shorter is better -- reviewers can read the diff.
WRITING PRINCIPLES
- Lead with value: the first sentence tells the reviewer *why this PR exists*, not *what files changed*.
- Describe the net result, not the journey: skip intermediate failures, debugging steps, and refactors done during development.
- Trust the final diff: if the goal or plan disagree with the diff, the diff is authoritative.
- Explain the non-obvious: spend description space on what the diff doesn't show -- why this approach, what was rejected, what to look at first.
- Use structure when it earns its keep: no empty sections, no template headers without content.
- If the body uses any `##` heading, the opening summary must also be under a heading (e.g. `## Summary`); otherwise a bare paragraph is fine.
PLAN SUMMARY
The full plan is attached separately as a <details> block, so do not restate it. Include a brief `### Plan Summary` with bullet points only when the change is medium or larger in the sizing matrix above. Skip it for small changes.
VISUAL AIDS
Include a visual aid only when a reviewer would struggle to reconstruct the mental model from prose alone -- based on what changes structurally, not on PR size. Skip for trivial / mechanical changes, or when prose already communicates clearly.
| PR changes... | Visual aid |
|---|---|
| 3+ interacting components or services | Mermaid component / interaction diagram |
| Multi-step workflow or pipeline with non-obvious sequencing | Mermaid flow diagram |
| 3+ behavioral modes or variants | Markdown comparison table |
| Before/after data or trade-offs | Markdown table |
| Data model changes with 3+ related entities | Mermaid ERD |
Mermaid: prefer `TB` direction, <=10 nodes typical. Place inline at the point of relevance, not in a separate "Diagrams" section.

712
lib/crates/fabro-workflow/src/pipeline/pull_request.rs
View file

@ -1,33 +1,133 @@
use std::sync::Arc;
use std::sync::{Arc, LazyLock};
use fabro_auth::CredentialSource;
use fabro_github::{self as github_app, ssh_url_to_https};
use fabro_graphviz::parser;
use fabro_llm::client::Client;
use fabro_llm::generate::{GenerateParams, generate};
use fabro_llm::generate::{GenerateParams, generate_object};
use fabro_model::Catalog;
use fabro_retro::retro::Retro;
use fabro_store::RunProjection;
use fabro_types::PullRequestRecord;
use fabro_types::settings::run::MergeStrategy;
use fabro_util::text::strip_goal_decoration;
use tracing::{debug, info};
use tracing::{debug, info, warn};
use super::types::{Concluded, Finalized, PullRequestOptions};
use crate::event::{Event, RunNoticeLevel};
use crate::event::{Event, RunNoticeCode, RunNoticeLevel};
use crate::outcome::{StageOutcome, format_cost as outcome_format_cost};
use crate::records::{Conclusion, RunSpec};
use crate::runtime_store::RunStoreHandle;
/// Maximum length of a PR title (Unicode scalar values).
const PR_TITLE_MAX_CHARS: usize = 72;
/// Structured output schema for the LLM-generated PR title and body.
static PR_CONTENT_SCHEMA: LazyLock<serde_json::Value> = LazyLock::new(|| {
serde_json::json!({
"type": "object",
"properties": {
"title": { "type": "string" },
"body": { "type": "string" }
},
"required": ["title", "body"],
"additionalProperties": false
})
});
/// Complete pull request content generated for a workflow run.
#[derive(Debug, serde::Deserialize)]
pub struct PrContent {
pub title: String,
pub body: String,
}
/// System prompt that instructs the LLM how to write a Fabro PR title and
/// body. The trailing programmatic sections (Plan `<details>`, Retro,
/// Fabro Details, footer) are appended after the LLM body — the prompt
/// explicitly forbids the LLM from duplicating them.
const PR_BODY_SYSTEM_PROMPT: &str = include_str!("prompts/pr_body.md");
const DEFAULT_PR_TITLE: &str = "Update workflow output";
const EMPTY_BODY_NOTICE: &str = "> _The LLM did not produce a description for this change. The diff and the appended details are the source of truth for review._";
/// Truncation budget for the LLM prompt's plan / diff sections.
#[derive(Debug, PartialEq, Eq)]
struct TruncationCaps {
plan: usize,
diff: usize,
}
const DIFF_HARD_CAP: usize = 500_000;
const PLAN_HARD_CAP: usize = 100_000;
const DIFF_FRACTION_NUM: usize = 4;
const PLAN_FRACTION_NUM: usize = 1;
const FRACTION_DEN: usize = 10;
const UNKNOWN_MODEL_CTX: usize = 200_000;
/// Resolve truncation caps based on the model's context window. Unknown
/// models use the baseline 200k context-window assumption.
fn truncation_caps(model: &str) -> TruncationCaps {
let ctx = Catalog::builtin()
.get(model)
.and_then(|m| usize::try_from(m.context_window()).ok())
.unwrap_or(UNKNOWN_MODEL_CTX);
truncation_caps_for_context_window(ctx)
}
fn truncation_caps_for_context_window(ctx: usize) -> TruncationCaps {
TruncationCaps {
diff: ctx
.saturating_mul(DIFF_FRACTION_NUM)
.checked_div(FRACTION_DEN)
.unwrap_or(DIFF_HARD_CAP)
.min(DIFF_HARD_CAP),
plan: ctx
.saturating_mul(PLAN_FRACTION_NUM)
.checked_div(FRACTION_DEN)
.unwrap_or(PLAN_HARD_CAP)
.min(PLAN_HARD_CAP),
}
}
/// Truncate `s` to at most `max` Unicode scalar values without splitting a
/// UTF-8 sequence.
fn truncate_chars(s: &str, max: usize) -> &str {
s.char_indices()
.nth(max)
.map_or(s, |(boundary, _)| &s[..boundary])
}
/// Truncate `s` to at most `max` Unicode scalar values, replacing the
/// trailing char with `…` when truncation occurs.
fn truncate_with_ellipsis(s: &str, max: usize) -> String {
if s.chars().count() > max {
let truncated: String = s.chars().take(max - 1).collect();
format!("{truncated}\u{2026}")
} else {
s.to_string()
}
}
/// Cap a PR title at [`PR_TITLE_MAX_CHARS`].
fn enforce_title_cap(title: &str) -> String {
truncate_with_ellipsis(title, PR_TITLE_MAX_CHARS)
}
/// Derive a PR title from the workflow goal.
///
/// Uses the first line, truncated to 120 characters for readability.
/// Uses the first line, truncated to the same cap as LLM-generated titles.
fn pr_title_from_goal(goal: &str) -> String {
let stripped = strip_goal_decoration(goal);
if stripped.chars().count() > 120 {
let truncated: String = stripped.chars().take(119).collect();
format!("{truncated}…")
truncate_with_ellipsis(strip_goal_decoration(goal), PR_TITLE_MAX_CHARS)
}
fn fallback_pr_title(goal: &str) -> String {
let title = pr_title_from_goal(goal);
if title.trim().is_empty() {
DEFAULT_PR_TITLE.to_string()
} else {
stripped.to_string()
title
}
}
@ -286,70 +386,43 @@ async fn load_pull_request_diff(run_store: &RunStoreHandle) -> String {
.unwrap_or_default()
}
/// Build a complete PR body by combining LLM-generated narrative with
/// programmatic sections (plan, retro, fabro details).
pub async fn build_pr_body(
/// Build complete PR content by combining LLM-generated narrative with
/// deterministic fallbacks and programmatic sections.
pub async fn build_pr_content(
diff: &str,
goal: &str,
model: &str,
run_store: &RunStoreHandle,
llm_source: &dyn CredentialSource,
conclusion: Option<&Conclusion>,
) -> Result<String, String> {
run_state: Option<&RunProjection>,
) -> Result<PrContent, String> {
let client = Client::from_source(llm_source)
.await
.map_err(|e| format!("Failed to create LLM client: {e}"))?;
build_pr_body_with_client(diff, goal, model, run_store, conclusion, Arc::new(client)).await
}
async fn build_pr_body_with_client(
diff: &str,
goal: &str,
model: &str,
run_store: &RunStoreHandle,
conclusion: Option<&Conclusion>,
client: Arc<Client>,
) -> Result<String, String> {
build_pr_body_with_client_and_state(diff, goal, model, run_store, conclusion, client, None)
.await
}
async fn build_pr_body_with_source_and_state(
diff: &str,
goal: &str,
model: &str,
run_store: &RunStoreHandle,
llm_source: &dyn CredentialSource,
conclusion: Option<&Conclusion>,
run_state: Option<&fabro_store::RunProjection>,
) -> Result<String, String> {
let client = Client::from_source(llm_source)
.await
.map_err(|e| format!("Failed to create LLM client: {e}"))?;
build_pr_body_with_client_and_state(
build_pr_content_with_client(
diff,
goal,
model,
run_store,
conclusion,
Arc::new(client),
run_state,
Arc::new(client),
)
.await
}
async fn build_pr_body_with_client_and_state(
async fn build_pr_content_with_client(
diff: &str,
goal: &str,
model: &str,
run_store: &RunStoreHandle,
conclusion: Option<&Conclusion>,
run_state: Option<&RunProjection>,
client: Arc<Client>,
run_state: Option<&fabro_store::RunProjection>,
) -> Result<String, String> {
info!("Building PR body");
) -> Result<PrContent, String> {
info!("Building PR content");
let loaded_run_state = if run_state.is_none() {
run_store
@ -369,29 +442,11 @@ async fn build_pr_body_with_client_and_state(
let run_spec = run_state.and_then(|state| state.spec.clone());
let dot_source = run_state.and_then(|state| state.graph_source.clone());
// Build LLM prompt
let system = if plan_text.is_some() {
"Write a PR description with: (1) 2-3 concise paragraphs explaining the change, then (2) a '### Plan Summary' section with bullet points summarizing the plan. Do not include a title. Do not include the full plan.".to_string()
} else {
"Write a concise PR description in 2-3 paragraphs explaining the change. Do not include a title.".to_string()
};
// Truncate diff to fit context windows (~50k chars)
let max_diff_len = 50_000;
let truncated_diff = if diff.len() > max_diff_len {
&diff[..diff.floor_char_boundary(max_diff_len)]
} else {
diff
};
let caps = truncation_caps(model);
let truncated_diff = truncate_chars(diff, caps.diff);
let prompt = if let Some(ref plan) = plan_text {
// Truncate plan for LLM context (~20k chars)
let max_plan_len = 20_000;
let truncated_plan = if plan.len() > max_plan_len {
&plan[..plan.floor_char_boundary(max_plan_len)]
} else {
plan.as_str()
};
let truncated_plan = truncate_chars(plan, caps.plan);
format!(
"Goal: {goal}\n\nPlan:\n```\n{truncated_plan}\n```\n\nDiff:\n```\n{truncated_diff}\n```"
)
@ -400,14 +455,32 @@ async fn build_pr_body_with_client_and_state(
};
let params = GenerateParams::new(model, client)
.system(system)
.system(PR_BODY_SYSTEM_PROMPT)
.prompt(prompt);
let result = generate(params)
let result = generate_object(params, PR_CONTENT_SCHEMA.clone())
.await
.map_err(|e| format!("LLM generation failed: {e}"))?;
let llm_output = result.response.text();
let output = result
.output
.ok_or_else(|| "LLM generation returned no structured output".to_string())?;
let generated: PrContent = serde_json::from_value(output)
.map_err(|e| format!("Failed to deserialize PR content: {e}"))?;
let title = if generated.title.trim().is_empty() {
fallback_pr_title(goal)
} else {
generated.title.trim().to_string()
};
let title = enforce_title_cap(&title);
let llm_body = if generated.body.trim().is_empty() {
warn!(model = %model, "LLM generated empty PR body; using skeleton PR body");
EMPTY_BODY_NOTICE.to_string()
} else {
generated.body
};
let retro_section = retro.as_ref().map(format_retro_section).unwrap_or_default();
let arc_details_section = conclusion
@ -416,15 +489,15 @@ async fn build_pr_body_with_client_and_state(
.unwrap_or_default();
let body = assemble_pr_body(
&llm_output,
&llm_body,
plan_text.as_deref(),
&retro_section,
&arc_details_section,
);
info!("PR body generated");
info!("PR content generated");
Ok(body)
Ok(PrContent { title, body })
}
/// Auto-merge configuration for a pull request.
@ -446,7 +519,7 @@ pub struct OpenPullRequestRequest<'a> {
pub run_store: &'a RunStoreHandle,
pub llm_source: &'a dyn CredentialSource,
pub conclusion: Option<&'a Conclusion>,
pub run_state: Option<&'a fabro_store::RunProjection>,
pub run_state: Option<&'a RunProjection>,
}
/// Optionally open a pull request after a successful workflow run.
@ -465,7 +538,7 @@ pub async fn maybe_open_pull_request(
let (owner, repo) =
github_app::parse_github_owner_repo(&https_url).map_err(|err| format!("{err:#}"))?;
let body = build_pr_body_with_source_and_state(
let content = build_pr_content(
req.diff,
req.goal,
req.model,
@ -476,9 +549,8 @@ pub async fn maybe_open_pull_request(
)
.await
.map_err(|err| format!("{err:#}"))?;
let body = truncate_pr_body(&body);
let title = pr_title_from_goal(req.goal);
let body = truncate_pr_body(&content.body);
let title = content.title;
let created = github_app::create_pull_request(
&req.github,
@ -603,7 +675,7 @@ pub async fn pull_request(concluded: Concluded, options: &PullRequestOptions) ->
.emit(&Event::PullRequestFailed { error: e.clone() });
services.emitter.notice(
RunNoticeLevel::Warn,
"pull_request_failed",
RunNoticeCode::PullRequestFailed,
format!("PR creation failed: {e}"),
);
}
@ -782,6 +854,16 @@ mod tests {
})
}
/// JSON string the MockProvider/openai mock returns to simulate the
/// structured-output response for `(title, body)`.
fn pr_content_json(title: &str, body: &str) -> String {
serde_json::to_string(&serde_json::json!({
"title": title,
"body": body,
}))
.unwrap()
}
fn make_test_conclusion() -> Conclusion {
Conclusion {
timestamp: Utc::now(),
@ -1123,20 +1205,25 @@ mod tests {
}
#[tokio::test]
async fn build_pr_body_uses_in_memory_conclusion() {
async fn build_pr_content_uses_in_memory_conclusion() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let body = build_pr_body_with_client(
let PrContent { title, body } = build_pr_content_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", "Narrative from mock."),
None,
explicit_client(
"mock",
&pr_content_json("Mock title", "Narrative from mock."),
),
)
.await
.unwrap();
assert_eq!(title, "Mock title");
assert!(body.contains("Narrative from mock."));
assert!(body.contains("### Fabro Details"));
assert!(body.contains("Ran 3 stages in 2m 30s for $0.42"));
@ -1144,7 +1231,7 @@ mod tests {
}
#[tokio::test]
async fn build_pr_body_uses_store_records_without_legacy_files() {
async fn build_pr_content_uses_store_records_without_legacy_files() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
@ -1196,16 +1283,21 @@ mod tests {
.await
.unwrap();
let body = build_pr_body_with_client(
let body = build_pr_content_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", "Narrative from mock."),
None,
explicit_client(
"mock",
&pr_content_json("Mock title", "Narrative from mock."),
),
)
.await
.unwrap();
.unwrap()
.body;
assert!(body.contains("Narrative from mock."));
assert!(body.contains("### Retro"));
@ -1214,7 +1306,7 @@ mod tests {
}
#[tokio::test]
async fn build_pr_body_uses_plan_text_from_store_without_response_md() {
async fn build_pr_content_uses_plan_text_from_store_without_response_md() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
@ -1283,42 +1375,52 @@ mod tests {
.await
.unwrap();
let body = build_pr_body_with_client(
let body = build_pr_content_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("mock", "Narrative from mock."),
None,
explicit_client(
"mock",
&pr_content_json("Mock title", "Narrative from mock."),
),
)
.await
.unwrap();
.unwrap()
.body;
assert!(body.contains("<summary>Full plan</summary>"));
assert!(body.contains("Plan from store"));
}
#[tokio::test]
async fn build_pr_body_uses_explicit_llm_client() {
async fn build_pr_content_uses_explicit_llm_client() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let body = build_pr_body_with_client(
let body = build_pr_content_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"gpt-5.4",
&run_store.clone().into(),
Some(&make_test_conclusion()),
explicit_client("openai", "Narrative from explicit client."),
None,
explicit_client(
"openai",
&pr_content_json("Explicit title", "Narrative from explicit client."),
),
)
.await
.unwrap();
.unwrap()
.body;
assert!(body.contains("Narrative from explicit client."));
assert!(!body.contains("Narrative from mock."));
}
#[tokio::test]
async fn build_pr_body_uses_vault_only_openai_codex_source() {
async fn build_pr_content_uses_vault_only_openai_codex_source() {
let server = MockServer::start_async().await;
let response_mock = server
.mock_async(|when, then| {
@ -1327,7 +1429,10 @@ mod tests {
.header("authorization", "Bearer vault-openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload("Narrative from vault source."));
.json_body(openai_responses_payload(&pr_content_json(
"Vault title",
"Narrative from vault source.",
)));
})
.await;
@ -1355,17 +1460,19 @@ mod tests {
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let run_store_handle: RunStoreHandle = run_store.into();
let body = build_pr_body(
let PrContent { title, body } = build_pr_content(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"gpt-5.4",
&run_store_handle,
llm_source.as_ref(),
Some(&make_test_conclusion()),
None,
)
.await
.unwrap();
assert_eq!(title, "Vault title");
assert!(body.contains("Narrative from vault source."));
response_mock.assert_async().await;
}
@ -1462,7 +1569,7 @@ mod tests {
fn pr_title_truncates_long_line() {
let long = "x".repeat(300);
let title = pr_title_from_goal(&long);
assert_eq!(title.chars().count(), 120);
assert_eq!(title.chars().count(), 72);
assert!(title.ends_with('…'));
}
@ -1485,6 +1592,42 @@ mod tests {
assert_eq!(pr_title_from_goal("Fix bug"), "Fix bug");
}
#[test]
fn truncation_caps_scale_with_context_window_and_clamp() {
assert_eq!(
truncation_caps_for_context_window(100_000),
TruncationCaps {
diff: 40_000,
plan: 10_000,
}
);
assert_eq!(
truncation_caps_for_context_window(200_000),
TruncationCaps {
diff: 80_000,
plan: 20_000,
}
);
assert_eq!(
truncation_caps_for_context_window(1_000_000),
TruncationCaps {
diff: 400_000,
plan: 100_000,
}
);
assert_eq!(
truncation_caps_for_context_window(10_000_000),
TruncationCaps {
diff: 500_000,
plan: 100_000,
}
);
assert_eq!(truncation_caps("unknown-model"), TruncationCaps {
diff: 80_000,
plan: 20_000,
});
}
#[tokio::test]
async fn empty_diff_returns_none() {
let store = test_store();
@ -1575,4 +1718,373 @@ mod tests {
assert!(diff.contains("from_store"));
}
// ── Structured-output PR content tests ──────────────────────────────
/// MockProvider returns an over-long title; builder must cap it at 72
/// chars and end with `…`. Exercises [`enforce_title_cap`] inside
/// [`build_pr_content_with_client`].
#[tokio::test]
async fn build_pr_content_truncates_long_title() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let long_title = "x".repeat(200);
let payload = pr_content_json(&long_title, "Body content.");
let title = build_pr_content_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
None,
explicit_client("mock", &payload),
)
.await
.unwrap()
.title;
assert_eq!(title.chars().count(), 72);
assert!(title.ends_with('\u{2026}'));
}
#[tokio::test]
async fn build_pr_content_uses_default_title_when_generated_and_goal_titles_empty() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let payload = pr_content_json("", "Body content.");
let title = build_pr_content_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
"## Plan:",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
None,
explicit_client("mock", &payload),
)
.await
.unwrap()
.title;
assert_eq!(title, DEFAULT_PR_TITLE);
}
/// Empty or whitespace-only bodies use the skeleton fallback instead of
/// aborting PR creation.
#[tokio::test]
async fn build_pr_content_uses_skeleton_when_body_empty() {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
let run_spec = RunSpec {
run_id: fixtures::RUN_1,
settings: fabro_types::WorkflowSettings::default(),
graph: Graph::new("test"),
workflow_slug: Some("test".to_string()),
source_directory: Some("/tmp/project".to_string()),
git: None,
labels: HashMap::new(),
provenance: None,
manifest_blob: None,
definition_blob: None,
fork_source_ref: None,
in_place: false,
};
append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated {
run_id: fixtures::RUN_1,
settings: serde_json::to_value(&run_spec.settings).unwrap(),
graph: serde_json::to_value(&run_spec.graph).unwrap(),
workflow_source: Some("digraph test { plan -> code }".to_string()),
workflow_config: None,
labels: run_spec.labels.clone().into_iter().collect(),
run_dir: "/tmp/project".to_string(),
source_directory: run_spec.source_directory.clone(),
workflow_slug: run_spec.workflow_slug.clone(),
db_prefix: None,
provenance: None,
manifest_blob: None,
git: None,
fork_source_ref: None,
in_place: false,
web_url: None,
})
.await
.unwrap();
append_event(&run_store, &fixtures::RUN_1, &Event::StageCompleted {
node_id: "plan".to_string(),
name: "plan".to_string(),
index: 0,
duration_ms: 1,
status: "succeeded".to_string(),
preferred_label: None,
suggested_next_ids: vec![],
billing: None,
failure: None,
notes: None,
files_touched: vec![],
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("Plan from store".to_string()),
attempt: 1,
max_attempts: 1,
})
.await
.unwrap();
append_event(&run_store, &fixtures::RUN_1, &Event::RetroCompleted {
duration_ms: 1,
response: Some(String::new()),
retro: Some(serde_json::to_value(make_test_retro()).unwrap()),
})
.await
.unwrap();
let payload = pr_content_json("Mock", " \n");
let body = build_pr_content_with_client(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
"Implement feature",
"mock-model",
&run_store.clone().into(),
Some(&make_test_conclusion()),
None,
explicit_client("mock", &payload),
)
.await
.unwrap()
.body;
assert!(body.contains("The LLM did not produce a description"));
assert!(body.contains("<summary>Full plan</summary>"));
assert!(body.contains("Plan from store"));
assert!(body.contains("### Retro"));
assert!(body.contains("### Fabro Details"));
assert!(body.contains("Generated with [Fabro](https://fabro.sh)"));
}
// ── maybe_open_pull_request fallback tests ──────────────────────────
/// Set of mock servers and credentials for the `maybe_open_pull_request`
/// fallback path. The builder's `Client::from_source` rebuilds the LLM
/// client from the credential source, so the in-process MockProvider
/// cannot intercept — we mock the OpenAI HTTP endpoint instead.
struct FallbackHarness {
_vault_dir: tempfile::TempDir,
// Held to keep the mock listener alive for the duration of the test;
// the test interacts with it via `Client::from_source` (which goes
// out via HTTP to the mock URL stored in `llm_source`).
openai_server: MockServer,
github_server: MockServer,
openai_mock_id: usize,
github_mock_id: usize,
llm_source: Arc<dyn CredentialSource>,
creds: fabro_github::GitHubCredentials,
run_store: RunStoreHandle,
}
impl FallbackHarness {
async fn assert_mocks_called_once(&self) {
httpmock::Mock::new(self.openai_mock_id, &self.openai_server)
.assert_async()
.await;
httpmock::Mock::new(self.github_mock_id, &self.github_server)
.assert_async()
.await;
}
}
/// Stand up an OpenAI mock that returns the given structured-output
/// payload, a GitHub mock that accepts a PR creation, a vault-backed
/// credential source, and a run store seeded with a non-empty
/// `final_patch`.
async fn setup_fallback_test_harness(openai_payload_text: &str) -> FallbackHarness {
let openai_server = MockServer::start_async().await;
let openai_mock = openai_server
.mock_async(|when, then| {
when.method(POST)
.path("/v1/responses")
.header("authorization", "Bearer vault-openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload(openai_payload_text));
})
.await;
let github_server = MockServer::start_async().await;
let github_mock = github_server
.mock_async(|when, then| {
when.method(POST)
.path("/repos/owner/repo/pulls")
.header("authorization", "Bearer test-token");
then.status(201)
.header("content-type", "application/json")
.json_body(serde_json::json!({
"number": 1,
"html_url": "https://example.test/owner/repo/pull/1",
"node_id": "PR_kwTest1",
}));
})
.await;
let vault_dir = tempfile::tempdir().unwrap();
let mut vault = Vault::load(vault_dir.path().join("secrets.json")).unwrap();
vault
.set(
"openai_codex",
&serde_json::to_string(&openai_api_key_credential("vault-openai-key")).unwrap(),
SecretType::Credential,
None,
)
.unwrap();
let base_url = openai_server.url("/v1");
let llm_source: Arc<dyn CredentialSource> =
Arc::new(VaultCredentialSource::with_env_lookup(
Arc::new(AsyncRwLock::new(vault)),
move |name| match name {
"OPENAI_BASE_URL" => Some(base_url.clone()),
_ => None,
},
));
let creds = fabro_github::GitHubCredentials::Token("test-token".to_string());
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
// Seed a non-empty `final_patch` so `load_pull_request_diff` returns
// diff content and the early-return for empty diffs does not fire.
let run_spec = RunSpec {
run_id: fixtures::RUN_1,
settings: fabro_types::WorkflowSettings::default(),
graph: Graph::new("test"),
workflow_slug: None,
source_directory: None,
git: None,
labels: HashMap::new(),
provenance: None,
manifest_blob: None,
definition_blob: None,
fork_source_ref: None,
in_place: false,
};
append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated {
run_id: fixtures::RUN_1,
settings: serde_json::to_value(&run_spec.settings).unwrap(),
graph: serde_json::to_value(&run_spec.graph).unwrap(),
workflow_source: None,
workflow_config: None,
labels: run_spec.labels.clone().into_iter().collect(),
run_dir: "/tmp/x".to_string(),
source_directory: None,
workflow_slug: None,
db_prefix: None,
provenance: None,
manifest_blob: None,
git: None,
fork_source_ref: None,
in_place: false,
web_url: None,
})
.await
.unwrap();
append_event(&run_store, &fixtures::RUN_1, &Event::WorkflowRunCompleted {
duration_ms: 1,
artifact_count: 0,
status: "succeeded".to_string(),
reason: SuccessReason::Completed,
total_usd_micros: None,
final_git_commit_sha: None,
final_patch: Some(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn from_store() {}\n".to_string(),
),
billing: None,
})
.await
.unwrap();
let openai_mock_id = openai_mock.id;
let github_mock_id = github_mock.id;
FallbackHarness {
_vault_dir: vault_dir,
openai_server,
github_server,
openai_mock_id,
github_mock_id,
llm_source,
creds,
run_store: run_store.into(),
}
}
/// LLM returns a usable body but an empty title; the content builder
/// falls back to `pr_title_from_goal` (first line, decoration stripped)
/// and PR creation succeeds with that title.
#[tokio::test]
async fn maybe_open_pull_request_falls_back_to_goal_title_when_llm_returns_empty_title() {
let payload = pr_content_json("", "Narrative.");
let harness = setup_fallback_test_harness(&payload).await;
let github_base_url = harness.github_server.url("");
let github = github_app::GitHubContext::new(&harness.creds, &github_base_url);
let result = maybe_open_pull_request(OpenPullRequestRequest {
github,
origin_url: "https://github.com/owner/repo.git",
base_branch: "main",
head_branch: "fabro/run/123",
goal: "Fix telemetry leak\n\ndetails...",
diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
model: "gpt-5.4",
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
conclusion: None,
run_state: None,
})
.await
.expect("PR creation should succeed");
let record = result.expect("PR record should be Some");
assert_eq!(record.title, "Fix telemetry leak");
harness.assert_mocks_called_once().await;
}
/// LLM returns an empty title; the content builder fallback still caps
/// the deterministic goal title at 72 chars ending with `…`.
#[tokio::test]
async fn maybe_open_pull_request_caps_fallback_title_at_72_chars() {
let payload = pr_content_json("", "Narrative.");
let harness = setup_fallback_test_harness(&payload).await;
let github_base_url = harness.github_server.url("");
let github = github_app::GitHubContext::new(&harness.creds, &github_base_url);
// Single ~200-char line, no `Plan:` / heading prefix, no newlines.
let goal = "x".repeat(200);
let result = maybe_open_pull_request(OpenPullRequestRequest {
github,
origin_url: "https://github.com/owner/repo.git",
base_branch: "main",
head_branch: "fabro/run/123",
goal: &goal,
diff: "diff --git a/src/lib.rs b/src/lib.rs\n+fn x() {}\n",
model: "gpt-5.4",
draft: false,
auto_merge: None,
run_store: &harness.run_store,
llm_source: harness.llm_source.as_ref(),
conclusion: None,
run_state: None,
})
.await
.expect("PR creation should succeed");
let record = result.expect("PR record should be Some");
assert_eq!(record.title.chars().count(), 72);
assert!(record.title.ends_with('\u{2026}'));
harness.assert_mocks_called_once().await;
}
}

2
lib/crates/fabro-workflow/src/pipeline/retro.rs
View file

@ -53,7 +53,7 @@ pub async fn run_retro(options: &RetroOptions, dry_run: bool) -> Option<Retro> {
return None;
}
};
let stage_durations = crate::extract_stage_durations_from_events(&events);
let stage_durations = crate::latest_stage_duration_by_node(&events);
let mut retro = derive_retro(
options.run_id,
&options.workflow_name,

2
lib/crates/fabro-workflow/src/pull_request.rs
View file

@ -1,4 +1,4 @@
pub use crate::pipeline::{
AutoMergeOptions, OpenPullRequestRequest, PullRequestRecord, build_pr_body,
AutoMergeOptions, OpenPullRequestRequest, PrContent, PullRequestRecord, build_pr_content,
maybe_open_pull_request,
};

7
lib/crates/fabro-workflow/src/test_support.rs
View file

@ -17,7 +17,7 @@ use crate::handler::HandlerRegistry;
use crate::outcome::Outcome;
use crate::pipeline;
use crate::pipeline::types::{Executed, Initialized};
use crate::pipeline::{billing_from_checkpoint, build_terminal_event};
use crate::pipeline::{billing_from_projection, build_terminal_event};
use crate::records::Checkpoint;
use crate::run_metadata::RunMetadataRuntime;
use crate::run_options::RunOptions;
@ -36,10 +36,7 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed {
let executed = Box::pin(pipeline::execute(initialized.initialized)).await;
initialized.store_logger.flush().await;
let state = executed.engine.run.run_store.state().await.ok();
let billing = state
.as_ref()
.and_then(|s| s.checkpoint.as_ref())
.and_then(billing_from_checkpoint);
let billing = state.as_ref().and_then(billing_from_projection);
let event = build_terminal_event(
&executed.outcome,
executed.duration_ms,

16
lib/crates/fabro-workflow/tests/it/integration.rs
View file

@ -6225,6 +6225,8 @@ mod real_llm {
_node: &Node,
prompt: &str,
_system_prompt: Option<&str>,
_emitter: &Arc<Emitter>,
_stage_scope: &fabro_workflow::event::StageScope,
) -> Result<CodergenResult, Error> {
self.complete(prompt).await
}
@ -6813,7 +6815,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
.header("authorization", "Bearer vault-openai-key");
then.status(200)
.header("content-type", "application/json")
.json_body(openai_responses_payload("Narrative from vault source."));
.json_body(openai_responses_payload(
&serde_json::to_string(&serde_json::json!({
"title": "Vault title",
"body": "Narrative from vault source.",
}))
.unwrap(),
));
})
.await;
@ -6894,7 +6902,7 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
let run_store = store.open_run_reader(&run_options.run_id).await.unwrap();
let run_store_handle: fabro_workflow::runtime_store::RunStoreHandle = run_store.into();
let body = fabro_workflow::pull_request::build_pr_body(
let content = fabro_workflow::pull_request::build_pr_content(
"diff --git a/src/lib.rs b/src/lib.rs\n+fn new_feature() {}\n",
"Implement feature",
"gpt-5.4",
@ -6910,11 +6918,13 @@ async fn workflow_run_with_vault_only_openai_codex_builds_pr_body() {
billing: None,
total_retries: 0,
}),
None,
)
.await
.expect("PR body should build from vault-only credentials");
assert!(body.contains("Narrative from vault source."));
assert_eq!(content.title, "Vault title");
assert!(content.body.contains("Narrative from vault source."));
response_mock.assert_async().await;
}

6
lib/packages/fabro-api-client/src/.openapi-generator/FILES
View file

@ -30,7 +30,6 @@ models/artifact-batch-upload-manifest.ts
models/artifact-entry.ts
models/artifact-list-response.ts
models/artifacts-settings.ts
models/assistant-stage-turn.ts
models/auth-method.ts
models/billed-token-counts.ts
models/billing-by-model.ts
@ -168,7 +167,6 @@ models/paginated-run-file-list.ts
models/paginated-run-list.ts
models/paginated-run-stage-list.ts
models/paginated-saved-query-list.ts
models/paginated-stage-turn-list.ts
models/pagination-meta.ts
models/pending-interview-record.ts
models/pre-run-push-outcome-failed.ts
@ -300,7 +298,6 @@ models/stage-completion.ts
models/stage-outcome.ts
models/stage-projection.ts
models/stage-state.ts
models/stage-turn.ts
models/start-run-request.ts
models/submit-answer-request.ts
models/success-reason.ts
@ -308,13 +305,10 @@ models/system-actor-kind.ts
models/system-features.ts
models/system-info-response.ts
models/system-run-counts.ts
models/system-stage-turn.ts
models/teams-integration-settings.ts
models/terminal-status.ts
models/timeline-entry-response.ts
models/tls-mode.ts
models/tool-stage-turn.ts
models/tool-use.ts
models/user-response.ts
models/validate-response.ts
models/webhook-strategy.ts

64
lib/packages/fabro-api-client/src/api/run-internals-api.ts
View file

@ -36,8 +36,6 @@ import type { PaginatedEventList } from '../models';
// @ts-ignore
import type { PaginatedRunStageList } from '../models';
// @ts-ignore
import type { PaginatedStageTurnList } from '../models';
// @ts-ignore
import type { RunArtifactListResponse } from '../models';
// @ts-ignore
import type { RunCheckpoint } from '../models';
@ -527,21 +525,21 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config
};
},
/**
* Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations.
* @summary List Stage Turns
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [pageLimit] Maximum number of items to return per page.
* @param {number} [pageOffset] Number of items to skip before returning results.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
listStageTurns: async (id: string, stageId: string, pageLimit?: number, pageOffset?: number, options: RawAxiosRequestConfig = {}): Promise<RequestArgs> => {
listStageEvents: async (id: string, stageId: string, sinceSeq?: number, limit?: number, options: RawAxiosRequestConfig = {}): Promise<RequestArgs> => {
// verify required parameter 'id' is not null or undefined
assertParamExists('listStageTurns', 'id', id)
assertParamExists('listStageEvents', 'id', id)
// verify required parameter 'stageId' is not null or undefined
assertParamExists('listStageTurns', 'stageId', stageId)
const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/turns`
assertParamExists('listStageEvents', 'stageId', stageId)
const localVarPath = `/api/v1/runs/{id}/stages/{stageId}/events`
.replace(`{${"id"}}`, encodeURIComponent(String(id)))
.replace(`{${"stageId"}}`, encodeURIComponent(String(stageId)));
// use dummy base URL string because the URL constructor only accepts absolute URLs.
@ -561,12 +559,12 @@ export const RunInternalsApiAxiosParamCreator = function (configuration?: Config
// http bearer authentication required
await setBearerAuthToObject(localVarHeaderParameter, configuration)
if (pageLimit !== undefined) {
localVarQueryParameter['page[limit]'] = pageLimit;
if (sinceSeq !== undefined) {
localVarQueryParameter['since_seq'] = sinceSeq;
}
if (pageOffset !== undefined) {
localVarQueryParameter['page[offset]'] = pageOffset;
if (limit !== undefined) {
localVarQueryParameter['limit'] = limit;
}
localVarHeaderParameter['Accept'] = 'application/json';
@ -964,19 +962,19 @@ export const RunInternalsApiFp = function(configuration?: Configuration) {
return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
},
/**
* Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations.
* @summary List Stage Turns
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [pageLimit] Maximum number of items to return per page.
* @param {number} [pageOffset] Number of items to skip before returning results.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<PaginatedStageTurnList>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.listStageTurns(id, stageId, pageLimit, pageOffset, options);
async listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<PaginatedEventList>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.listStageEvents(id, stageId, sinceSeq, limit, options);
const localVarOperationServerIndex = configuration?.serverIndex ?? 0;
const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageTurns']?.[localVarOperationServerIndex]?.url;
const localVarOperationServerBasePath = operationServerMap['RunInternalsApi.listStageEvents']?.[localVarOperationServerIndex]?.url;
return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath);
},
/**
@ -1174,17 +1172,17 @@ export const RunInternalsApiFactory = function (configuration?: Configuration, b
return localVarFp.listStageArtifacts(id, stageId, options).then((request) => request(axios, basePath));
},
/**
* Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations.
* @summary List Stage Turns
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [pageLimit] Maximum number of items to return per page.
* @param {number} [pageOffset] Number of items to skip before returning results.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig): AxiosPromise<PaginatedStageTurnList> {
return localVarFp.listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(axios, basePath));
listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig): AxiosPromise<PaginatedEventList> {
return localVarFp.listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(axios, basePath));
},
/**
* Uploads one or more artifacts for a stage. Intended for trusted internal callers. The server accepts both: - `application/octet-stream` for single-file uploads with the `filename` query parameter - strict manifest-first `multipart/form-data` uploads documented by `ArtifactBatchUploadManifest` The generated Rust client currently exposes the octet-stream variant because the OpenAPI code generator in this repo does not support multiple request media types on one operation.
@ -1374,17 +1372,17 @@ export class RunInternalsApi extends BaseAPI {
}
/**
* Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations.
* @summary List Stage Turns
* Returns a paginated JSON list of stored run events scoped to a single stage visit.
* @summary List Stage Events
* @param {string} id Unique run identifier (ULID).
* @param {string} stageId Identifier of a stage within a run\&#39;s workflow graph, serialized as &#x60;node_id@visit&#x60;.
* @param {number} [pageLimit] Maximum number of items to return per page.
* @param {number} [pageOffset] Number of items to skip before returning results.
* @param {number} [sinceSeq] First event sequence number to include.
* @param {number} [limit] Maximum number of events to return.
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
public listStageTurns(id: string, stageId: string, pageLimit?: number, pageOffset?: number, options?: RawAxiosRequestConfig) {
return RunInternalsApiFp(this.configuration).listStageTurns(id, stageId, pageLimit, pageOffset, options).then((request) => request(this.axios, this.basePath));
public listStageEvents(id: string, stageId: string, sinceSeq?: number, limit?: number, options?: RawAxiosRequestConfig) {
return RunInternalsApiFp(this.configuration).listStageEvents(id, stageId, sinceSeq, limit, options).then((request) => request(this.axios, this.basePath));
}
/**

34
lib/packages/fabro-api-client/src/models/assistant-stage-turn.ts
View file

@ -1,34 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.1.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
/**
* An assistant response turn within a stage.
*/
export interface AssistantStageTurn {
'kind': AssistantStageTurnKindEnum;
/**
* Assistant response text.
*/
'content': string;
}
export const AssistantStageTurnKindEnum = {
ASSISTANT: 'assistant'
} as const;
export type AssistantStageTurnKindEnum = typeof AssistantStageTurnKindEnum[keyof typeof AssistantStageTurnKindEnum];

2
lib/packages/fabro-api-client/src/models/billing-by-model.ts
View file

@ -26,7 +26,7 @@ import type { ModelReference } from './model-reference';
export interface BillingByModel {
'model': ModelReference;
/**
* Number of stages that used this model.
* Number of usage-bearing stage visits that used this model.
*/
'stages': number;
'billing': BilledTokenCounts;

2
lib/packages/fabro-api-client/src/models/billing-stage-ref.ts
View file

@ -15,7 +15,7 @@
/**
* Reference to a billing stage.
* Reference to a workflow node in a billing stage row.
*/
export interface BillingStageRef {
/**

6
lib/packages/fabro-api-client/src/models/board-column-definition.ts
View file

@ -13,9 +13,11 @@
*/
// May contain unused imports in some cases
// @ts-ignore
import type { BoardColumn } from './board-column';
export interface BoardColumnDefinition {
'id': string;
'id': BoardColumn;
'name': string;
}

1
lib/packages/fabro-api-client/src/models/board-column.ts
View file

@ -19,6 +19,7 @@
*/
export const BoardColumn = {
QUEUED: 'queued',
INITIALIZING: 'initializing',
RUNNING: 'running',
BLOCKED: 'blocked',

6
lib/packages/fabro-api-client/src/models/index.ts
View file

@ -10,7 +10,6 @@ export * from './artifact-batch-upload-manifest';
export * from './artifact-entry';
export * from './artifact-list-response';
export * from './artifacts-settings';
export * from './assistant-stage-turn';
export * from './auth-method';
export * from './billed-token-counts';
export * from './billing-by-model';
@ -147,7 +146,6 @@ export * from './paginated-run-file-list';
export * from './paginated-run-list';
export * from './paginated-run-stage-list';
export * from './paginated-saved-query-list';
export * from './paginated-stage-turn-list';
export * from './pagination-meta';
export * from './pending-interview-record';
export * from './pre-run-push-outcome';
@ -279,7 +277,6 @@ export * from './stage-completion';
export * from './stage-outcome';
export * from './stage-projection';
export * from './stage-state';
export * from './stage-turn';
export * from './start-run-request';
export * from './submit-answer-request';
export * from './success-reason';
@ -287,13 +284,10 @@ export * from './system-actor-kind';
export * from './system-features';
export * from './system-info-response';
export * from './system-run-counts';
export * from './system-stage-turn';
export * from './teams-integration-settings';
export * from './terminal-status';
export * from './timeline-entry-response';
export * from './tls-mode';
export * from './tool-stage-turn';
export * from './tool-use';
export * from './user-response';
export * from './validate-response';
export * from './webhook-strategy';

41
lib/packages/fabro-api-client/src/models/manifest-git.ts
View file

@ -1,41 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.1.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
// May contain unused imports in some cases
// @ts-ignore
import type { ManifestPreRunPushOutcome } from './manifest-pre-run-push-outcome';
/**
* Observable git state from the CLI working directory.
*/
export interface ManifestGit {
/**
* Remote origin URL with any embedded credentials removed.
*/
'origin_url': string;
/**
* Current branch name.
*/
'branch': string;
/**
* Current commit SHA.
*/
'sha': string;
/**
* Whether the working tree has uncommitted changes.
*/
'clean': boolean;
'push_outcome': ManifestPreRunPushOutcome;
}

36
lib/packages/fabro-api-client/src/models/manifest-pre-run-push-outcome.ts
View file

@ -1,36 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.1.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
/**
* Outcome of the CLI\'s best-effort pre-run push.
*/
export interface ManifestPreRunPushOutcome {
'type': ManifestPreRunPushOutcomeTypeEnum;
'remote'?: string | null;
'branch'?: string | null;
'message'?: string | null;
'repo_origin_url'?: string | null;
}
export const ManifestPreRunPushOutcomeTypeEnum = {
NOT_ATTEMPTED: 'not_attempted',
SUCCEEDED: 'succeeded',
FAILED: 'failed',
SKIPPED_NO_REMOTE: 'skipped_no_remote',
SKIPPED_REMOTE_MISMATCH: 'skipped_remote_mismatch'
} as const;
export type ManifestPreRunPushOutcomeTypeEnum = typeof ManifestPreRunPushOutcomeTypeEnum[keyof typeof ManifestPreRunPushOutcomeTypeEnum];

30
lib/packages/fabro-api-client/src/models/paginated-stage-turn-list.ts
View file

@ -1,30 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.1.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
// May contain unused imports in some cases
// @ts-ignore
import type { PaginationMeta } from './pagination-meta';
// May contain unused imports in some cases
// @ts-ignore
import type { StageTurn } from './stage-turn';
/**
* Paginated list of stage turns.
*/
export interface PaginatedStageTurnList {
'data': Array<StageTurn>;
'meta': PaginationMeta;
}

30
lib/packages/fabro-api-client/src/models/pre-run-git-context.ts
View file

@ -1,30 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.1.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
// May contain unused imports in some cases
// @ts-ignore
import type { DirtyStatus } from './dirty-status';
// May contain unused imports in some cases
// @ts-ignore
import type { PreRunPushOutcome } from './pre-run-push-outcome';
/**
* Submitter-side git context captured before run creation.
*/
export interface PreRunGitContext {
'display_base_sha'?: string | null;
'local_dirty': DirtyStatus;
'push_outcome': PreRunPushOutcome;
}

13
lib/packages/fabro-api-client/src/models/run-billing-stage.ts
View file

@ -22,17 +22,24 @@ import type { BillingStageRef } from './billing-stage-ref';
// May contain unused imports in some cases
// @ts-ignore
import type { ModelReference } from './model-reference';
// May contain unused imports in some cases
// @ts-ignore
import type { StageState } from './stage-state';
/**
* Token counts and billed totals for a single stage within a run.
* Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node.
*/
export interface RunBillingStage {
'stage': BillingStageRef;
'model': ModelReference | null;
'billing': BilledTokenCounts;
/**
* Wall-clock runtime in seconds.
* Wall-clock runtime in seconds, summed across every visit of this node.
*/
'runtime_secs': number;
/**
* Wall-clock time the latest attempt of this stage started, if known.
*/
'started_at'?: string | null;
'state'?: StageState | null;
}

2
lib/packages/fabro-api-client/src/models/run-billing.ts
View file

@ -28,7 +28,7 @@ import type { RunBillingTotals } from './run-billing-totals';
*/
export interface RunBilling {
/**
* Per-stage billing breakdown.
* Per-node billing breakdown. Each row sums billing and runtime across all visits of that node.
*/
'stages': Array<RunBillingStage>;
'totals': RunBillingTotals;

17
lib/packages/fabro-api-client/src/models/run-stage.ts
View file

@ -22,7 +22,7 @@ import type { StageState } from './stage-state';
*/
export interface RunStage {
/**
* Unique stage identifier within the run.
* StageId in \"node_id@visit\" form, e.g. verify@2.
*/
'id': string;
/**
@ -35,10 +35,15 @@ export interface RunStage {
*/
'duration_secs'?: number;
/**
* Node identifier in the Graphviz graph source.
* Node id in the workflow graph; multiple stages with different visits share the same node_id.
*/
'dot_id'?: string;
'node_id': string;
/**
* 1-based visit count; bumped each time the workflow re-enters this node.
*/
'visit': number;
/**
* Wall-clock time the latest attempt of this stage started, if known.
*/
'started_at'?: string | null;
}

25
lib/packages/fabro-api-client/src/models/set-secret-request.ts
View file

@ -1,25 +0,0 @@
/* tslint:disable */
/* eslint-disable */
/**
* Fabro Run API
* HTTP API for managing Fabro workflow run executions.
*
* The version of the OpenAPI document: 0.1.0
*
*
* NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
* https://openapi-generator.tech
* Do not edit the class manually.
*/
/**
* Request to store a secret value.
*/
export interface SetSecretRequest {
/**
* The secret value to store.
*/
'value': string;
}

Some files were not shown because too many files have changed in this diff Show more