scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-09-13 23:14:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge remote-tracking branch 'origin/main' into fabro/run/01KQT1TWWJYWZGDT8F05E29H9D

Browse source 
# Conflicts:
#	apps/fabro-web/app/lib/run-events.test.tsx
#	apps/fabro-web/app/lib/run-events.ts
#	lib/crates/fabro-cli/src/commands/run/runner.rs
#	lib/crates/fabro-server/src/server/handler/mod.rs
#	lib/crates/fabro-workflow/src/event/convert.rs
#	lib/crates/fabro-workflow/src/event/events.rs
#	lib/crates/fabro-workflow/src/event/names.rs
#	lib/crates/fabro-workflow/src/event/stored_fields.rs
#	lib/crates/fabro-workflow/src/handler/llm/api.rs
This commit is contained in:
Bryan Helmkamp 2026-05-05 10:10:13 -04:00
commit b5e38d404a
No known key found for this signature in database
145 changed files with 10199 additions and 2367 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
Cargo.lock generated
View file

@ -1536,7 +1536,7 @@ dependencies = [
[[package]]
name = "fabro-agent"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1575,7 +1575,7 @@ dependencies = [
[[package]]
name = "fabro-api"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-config",
@ -1596,7 +1596,7 @@ dependencies = [
[[package]]
name = "fabro-auth"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1620,7 +1620,7 @@ dependencies = [
[[package]]
name = "fabro-checkpoint"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-config",
@ -1636,7 +1636,7 @@ dependencies = [
[[package]]
name = "fabro-cli"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -1732,7 +1732,7 @@ dependencies = [
[[package]]
name = "fabro-client"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"bytes",
@ -1761,7 +1761,7 @@ dependencies = [
[[package]]
name = "fabro-config"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"chrono",
@ -1788,7 +1788,7 @@ dependencies = [
[[package]]
name = "fabro-core"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"fabro-types",
@ -1803,7 +1803,7 @@ dependencies = [
[[package]]
name = "fabro-dev"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -1823,7 +1823,7 @@ dependencies = [
[[package]]
name = "fabro-devcontainer"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-http",
"fabro-static",
@ -1840,7 +1840,7 @@ dependencies = [
[[package]]
name = "fabro-dump"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"bytes",
@ -1854,7 +1854,7 @@ dependencies = [
[[package]]
name = "fabro-github"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -1876,7 +1876,7 @@ dependencies = [
[[package]]
name = "fabro-graphviz"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-types",
@ -1890,7 +1890,7 @@ dependencies = [
[[package]]
name = "fabro-hooks"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"fabro-agent",
@ -1914,7 +1914,7 @@ dependencies = [
[[package]]
name = "fabro-http"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-static",
"http",
@ -1924,7 +1924,7 @@ dependencies = [
[[package]]
name = "fabro-install"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -1939,7 +1939,7 @@ dependencies = [
[[package]]
name = "fabro-interview"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"dialoguer",
@ -1954,7 +1954,7 @@ dependencies = [
[[package]]
name = "fabro-llm"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -1986,7 +1986,7 @@ dependencies = [
[[package]]
name = "fabro-macros"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"clap",
"fabro-options-metadata",
@ -1997,7 +1997,7 @@ dependencies = [
[[package]]
name = "fabro-mcp"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-config",
@ -2013,7 +2013,7 @@ dependencies = [
[[package]]
name = "fabro-model"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-static",
"insta",
@ -2024,7 +2024,7 @@ dependencies = [
[[package]]
name = "fabro-oauth"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"axum",
@ -2046,7 +2046,7 @@ dependencies = [
[[package]]
name = "fabro-options-metadata"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"serde",
"serde_json",
@ -2054,7 +2054,7 @@ dependencies = [
[[package]]
name = "fabro-proc"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"cc",
"libc",
@ -2063,7 +2063,7 @@ dependencies = [
[[package]]
name = "fabro-redact"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"aho-corasick",
"ref-cast",
@ -2079,7 +2079,7 @@ dependencies = [
[[package]]
name = "fabro-retro"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"chrono",
@ -2098,7 +2098,7 @@ dependencies = [
[[package]]
name = "fabro-sandbox"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2138,7 +2138,7 @@ dependencies = [
[[package]]
name = "fabro-server"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2219,7 +2219,7 @@ dependencies = [
[[package]]
name = "fabro-slack"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-http",
"fabro-interview",
@ -2240,18 +2240,18 @@ dependencies = [
[[package]]
name = "fabro-spa"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"rust-embed",
]
[[package]]
name = "fabro-static"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
[[package]]
name = "fabro-store"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"async-trait",
"bytes",
@ -2278,7 +2278,7 @@ dependencies = [
[[package]]
name = "fabro-telemetry"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"base64",
@ -2304,7 +2304,7 @@ dependencies = [
[[package]]
name = "fabro-template"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"fabro-util",
@ -2316,7 +2316,7 @@ dependencies = [
[[package]]
name = "fabro-test"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"assert_cmd",
"axum",
@ -2339,7 +2339,7 @@ dependencies = [
[[package]]
name = "fabro-tracker"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-trait",
@ -2353,7 +2353,7 @@ dependencies = [
[[package]]
name = "fabro-types"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"clap",
@ -2374,7 +2374,7 @@ dependencies = [
[[package]]
name = "fabro-util"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"console 0.15.11",
@ -2394,7 +2394,7 @@ dependencies = [
[[package]]
name = "fabro-validate"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"fabro-graphviz",
"fabro-model",
@ -2404,7 +2404,7 @@ dependencies = [
[[package]]
name = "fabro-vault"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"chrono",
"fabro-types",
@ -2416,7 +2416,7 @@ dependencies = [
[[package]]
name = "fabro-workflow"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"assert_cmd",
@ -7120,7 +7120,7 @@ dependencies = [
[[package]]
name = "twin-github"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"axum",
"base64",
@ -7139,7 +7139,7 @@ dependencies = [
[[package]]
name = "twin-openai"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
dependencies = [
"anyhow",
"async-stream",

2
Cargo.toml
View file

@ -5,7 +5,7 @@ resolver = "2"
[workspace.package]
edition = "2021"
version = "0.223.0-nightly.0"
version = "0.224.0-nightly.0"
license = "MIT"
[workspace.dependencies]

20
apps/fabro-web/app/components/stage-sidebar.tsx
View file

@ -1,4 +1,4 @@
import { useState, useEffect, useRef, type ComponentType } from "react";
import { useEffect, useRef, type ComponentType } from "react";
import { Link } from "react-router";
import type { StageState } from "@qltysh/fabro-api-client";
import {
@ -11,14 +11,16 @@ import {
} from "@heroicons/react/24/solid";
import { Bars3BottomLeftIcon, DocumentTextIcon, MapIcon } from "@heroicons/react/24/outline";
import { formatDurationSecs } from "../lib/format";
import { ACTIVE_STAGE_STATES } from "../lib/stage-sidebar";
import { ACTIVE_STAGE_STATES, formatStageLabel } from "../lib/stage-sidebar";
import { useTickingNow } from "../lib/time";
export interface Stage {
id: string;
name: string;
status: StageState;
duration: string;
dotId?: string;
nodeId: string;
visit: number;
}
export const statusConfig: Record<StageState, { icon: ComponentType<{ className?: string }>; color: string }> = {
@ -42,7 +44,6 @@ interface StageSidebarProps {
export function StageSidebar({ stages, runId, selectedStageId, activeLink }: StageSidebarProps) {
// Track when we first observed each running stage (for ticking timer)
const runningStartRef = useRef<Map<string, number>>(new Map());
const [, setTick] = useState(0);
// Track start times for running stages
useEffect(() => {
@ -62,16 +63,13 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta
}, [stages]);
// Tick every second while any stage is running
useEffect(() => {
if (!stages.some((s) => ACTIVE_STAGE_STATES.has(s.status))) return;
const interval = setInterval(() => setTick((t) => t + 1), 1000);
return () => clearInterval(interval);
}, [stages]);
const hasActive = stages.some((s) => ACTIVE_STAGE_STATES.has(s.status));
const now = useTickingNow(hasActive);
function stageDuration(stage: Stage): string {
if (ACTIVE_STAGE_STATES.has(stage.status)) {
const start = runningStartRef.current.get(stage.id);
if (start) return formatDurationSecs(Math.floor((Date.now() - start) / 1000));
if (start) return formatDurationSecs(Math.floor((now - start) / 1000));
return "0s";
}
return stage.duration;
@ -100,7 +98,7 @@ export function StageSidebar({ stages, runId, selectedStageId, activeLink }: Sta
}`}
>
<Icon className={`size-4 shrink-0 ${config.color} ${ACTIVE_STAGE_STATES.has(stage.status) ? "animate-spin" : ""}`} />
<span className="flex-1 truncate">{stage.name}</span>
<span className="flex-1 truncate">{formatStageLabel(stage)}</span>
<span className="font-mono text-xs tabular-nums text-fg-muted">{stageDuration(stage)}</span>
</Link>
</li>

23
apps/fabro-web/app/data/runs.ts
View file

@ -1,8 +1,10 @@
import { formatElapsedSecs, formatDurationSecs } from "../lib/format";
import type {
RunListItem,
RunStatus as ApiRunStatus,
RunSummary,
import {
BoardColumn,
type BoardColumn as ApiBoardColumn,
type RunListItem,
type RunStatus as ApiRunStatus,
type RunSummary,
} from "@qltysh/fabro-api-client";
export type CiStatus = "passing" | "failing" | "pending";
@ -37,9 +39,19 @@ export interface RunItem {
sourceDirectory?: string;
}
export type ColumnStatus = "initializing" | "running" | "blocked" | "succeeded" | "failed";
export type ColumnStatus = ApiBoardColumn;
export const columnStatuses = [
BoardColumn.QUEUED,
BoardColumn.INITIALIZING,
BoardColumn.RUNNING,
BoardColumn.BLOCKED,
BoardColumn.SUCCEEDED,
BoardColumn.FAILED,
] as const satisfies readonly ColumnStatus[];
export const columnStatusDisplay: Record<ColumnStatus, { label: string; dot: string; text: string }> = {
queued: { label: "Queued", dot: "bg-fg-muted", text: "text-fg-muted" },
initializing: { label: "Initializing", dot: "bg-amber", text: "text-amber" },
running: { label: "Running", dot: "bg-teal-500", text: "text-teal-500" },
blocked: { label: "Blocked", dot: "bg-amber", text: "text-amber" },
@ -113,6 +125,7 @@ export function columnForStatus(status: ApiRunStatus | null | undefined): Column
switch (status?.kind) {
case "submitted":
case "queued":
return "queued";
case "starting":
return "initializing";
case "running":

65
apps/fabro-web/app/lib/api-client.ts
View file

@ -217,6 +217,69 @@ export async function apiPaginatedFetcher<TItem, TExtra extends object = {}>(
}
}
function stageEventsPagePath(key: string, sinceSeq: number, limit: number): string {
const url = new URL(apiPath(key), "http://fabro.local");
url.searchParams.set("since_seq", String(sinceSeq));
url.searchParams.set("limit", String(limit));
return `${url.pathname}${url.search}`;
}
/**
* Cursor-paginated fetcher for `/runs/{id}/stages/{stageId}/events`.
*
* Loops from `since_seq=1` with a 1000-event page size, advancing the cursor
* to `highestSeq + 1` until the server reports `meta.has_more === false`.
* The empty-page guard mirrors `apiPaginatedFetcher`: if the server claims
* `has_more` but returns no rows we exit and `console.warn` to surface the
* server invariant violation without spinning the UI.
*/
export async function fetchAllStageEvents<TItem extends { seq: number }>(
key: string,
): Promise<TItem[]> {
const PAGE_LIMIT = 1000;
const MAX_PAGES = 50;
const data: TItem[] = [];
let sinceSeq = 1;
let pagesLoaded = 0;
while (true) {
const response = await apiRequest(stageEventsPagePath(key, sinceSeq, PAGE_LIMIT));
if (!response.ok) {
throw await apiErrorFromResponse(response);
}
const page = (await response.json()) as PaginatedEnvelope<TItem>;
pagesLoaded += 1;
if (page.data.length === 0) {
if (page.meta.has_more) {
console.warn(
`Stage events fetch for ${key} returned an empty page with has_more=true; stopping at ${data.length} items to avoid spinning.`,
);
}
return data;
}
data.push(...page.data);
if (!page.meta.has_more) return data;
if (pagesLoaded >= MAX_PAGES) {
console.warn(
`Stopped stage events fetch for ${key} after ${pagesLoaded} pages and ${data.length} items because the safety cap was reached.`,
);
return data;
}
const highestSeq = page.data.reduce((max, event) => Math.max(max, event.seq), sinceSeq - 1);
if (highestSeq < sinceSeq) {
console.warn(
`Stage events fetch for ${key} returned a non-advancing page at since_seq=${sinceSeq}; stopping at ${data.length} items to avoid spinning.`,
);
return data;
}
sinceSeq = highestSeq + 1;
}
}
export async function apiJsonMutation<TResponse, TArg = unknown>(
key: string,
{ arg }: { arg: TArg },
@ -233,4 +296,4 @@ export async function apiJsonMutation<TResponse, TArg = unknown>(
}
if (response.status === 204) return undefined as TResponse;
return response.json() as Promise<TResponse>;
}
}

94
apps/fabro-web/app/lib/board-events.test.tsx
View file

@ -4,7 +4,12 @@ import {
shouldRefreshBoardForEvent,
subscribeToBoardEvents,
} from "./board-events";
import {
createCrossTabSseCoordinator,
type BroadcastChannelLike,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import type { EventSourceLike } from "./sse";
type MessageHandler = ((event: { data: string }) => void) | null;
@ -21,6 +26,14 @@ class FakeEventSource {
}
}
class FakeBroadcastChannel implements BroadcastChannelLike {
onmessage: ((event: { data: unknown }) => void) | null = null;
postMessage() {}
close() {}
}
describe("shouldRefreshBoardForEvent", () => {
test("refreshes board for run and interview status changes only", () => {
expect(shouldRefreshBoardForEvent("run.running")).toBe(true);
@ -31,22 +44,28 @@ describe("shouldRefreshBoardForEvent", () => {
});
describe("subscribeToBoardEvents", () => {
test("shares one source and invalidates the board runs key", () => {
test("coordinated mode shares one global source and invalidates the board runs key", async () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createCoordinator((url) => {
created.push(url);
return source;
});
const mutate = (key: string) => {
keys.push(key);
return Promise.resolve();
};
const firstCleanup = subscribeToBoardEvents(mutate, (url) => {
created.push(url);
return source;
}, { debounceMs: 0 });
const firstCleanup = subscribeToBoardEvents(mutate, () => {
throw new Error("source should be created by coordinator");
}, { debounceMs: 0, coordinator });
const secondCleanup = subscribeToBoardEvents(mutate, () => {
throw new Error("source should be reused");
}, { debounceMs: 0 });
}, { debounceMs: 0, coordinator });
await waitFor(() => created.length === 1);
keys.length = 0;
source.emit({ event: "run.running" });
@ -57,5 +76,68 @@ describe("subscribeToBoardEvents", () => {
expect(source.closed).toBe(false);
secondCleanup();
expect(source.closed).toBe(true);
coordinator.close();
});
test("fallback mode preserves the existing shared board EventSource", () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const mutate = (key: string) => {
keys.push(key);
return Promise.resolve();
};
const firstCleanup = subscribeToBoardEvents(mutate, (url) => {
created.push(url);
return source;
}, { debounceMs: 0, coordinator });
const secondCleanup = subscribeToBoardEvents(mutate, () => {
throw new Error("source should be reused");
}, { debounceMs: 0, coordinator });
source.emit({ event: "run.running" });
expect(created).toEqual(["/api/v1/attach"]);
expect(keys).toEqual([queryKeys.boards.runs()]);
firstCleanup();
expect(source.closed).toBe(false);
secondCleanup();
expect(source.closed).toBe(true);
coordinator.close();
});
});
function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) {
return createCrossTabSseCoordinator({
tabId: "board-test",
channelFactory: () => new FakeBroadcastChannel(),
eventSourceFactory,
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: {
heartbeatMs: 10,
leaderStaleMs: 50,
electionJitterMs: 0,
},
});
}
function createFallbackCoordinator() {
return createCrossTabSseCoordinator({
channelFactory: () => {
throw new Error("BroadcastChannel unavailable");
},
});
}
async function waitFor(condition: () => boolean, timeoutMs = 200) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (condition()) return;
await new Promise((resolve) => setTimeout(resolve, 2));
}
throw new Error("condition did not become true before timeout");
}

42
apps/fabro-web/app/lib/board-events.ts
View file

@ -1,6 +1,10 @@
import { useEffect } from "react";
import { useSWRConfig } from "swr";
import {
subscribeToCrossTabSse,
type CrossTabSseCoordinator,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import {
createBrowserEventSource,
@ -11,6 +15,11 @@ import {
type SharedEventSubscription,
} from "./sse";
interface BoardEventOptions {
debounceMs?: number;
coordinator?: CrossTabSseCoordinator;
}
const BOARD_STATUS_EVENTS = new Set([
"run.submitted",
"run.queued",
@ -41,23 +50,36 @@ export function shouldRefreshBoardForEvent(event: string) {
export function subscribeToBoardEvents(
mutate: MutateFn,
eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource,
{ debounceMs = 500 }: { debounceMs?: number } = {},
{ debounceMs = 500, coordinator }: BoardEventOptions = {},
): () => void {
return subscribeToSharedEventSource<EventPayload>({
subscriptions,
return subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey: BOARD_SUBSCRIPTION_KEY,
url: queryKeys.system.attach(),
mutate,
eventSourceFactory,
debounceMs,
resolveInvalidation: (payload) => ({
keys: payload.event && shouldRefreshBoardForEvent(payload.event)
? [queryKeys.boards.runs()]
: [],
}),
resyncKeys: () => [queryKeys.boards.runs()],
resolveInvalidation: boardInvalidation,
fallbackSubscribe: () =>
subscribeToSharedEventSource<EventPayload>({
subscriptions,
subscriptionKey: BOARD_SUBSCRIPTION_KEY,
url: queryKeys.system.attach(),
mutate,
eventSourceFactory,
debounceMs,
resolveInvalidation: boardInvalidation,
}),
});
}
function boardInvalidation(payload: EventPayload) {
return {
keys: payload.event && shouldRefreshBoardForEvent(payload.event)
? [queryKeys.boards.runs()]
: [],
};
}
export function useBoardEvents() {
const { mutate } = useSWRConfig();

736
apps/fabro-web/app/lib/cross-tab-sse.test.ts Normal file
View file

@ -0,0 +1,736 @@
import { afterEach, describe, expect, test } from "bun:test";
import {
CROSS_TAB_SSE_CHANNEL,
createCrossTabSseCoordinator,
subscribeToCrossTabSse,
type BroadcastChannelLike,
type CrossTabSseCoordinator,
type CrossTabSseMessage,
} from "./cross-tab-sse";
import type { EventPayload, MutateFn } from "./sse";
type MessageHandler = ((event: { data: string }) => void) | null;
type TabVisibility = "visible" | "hidden";
const TEST_TIMING = {
heartbeatMs: 10,
leaderStaleMs: 35,
electionJitterMs: 5,
};
class FakeEventSource {
onmessage: MessageHandler = null;
closed = false;
constructor(
readonly url: string,
readonly owner: string,
) {}
emit(payload: unknown) {
this.onmessage?.({ data: JSON.stringify(payload) });
}
close() {
this.closed = true;
}
}
class FakeBroadcastChannel implements BroadcastChannelLike {
static channels = new Set<FakeBroadcastChannel>();
static muted = false;
static throwOnTypes = new Set<CrossTabSseMessage["type"]>();
onmessage: ((event: { data: unknown }) => void) | null = null;
closed = false;
constructor(readonly name: string) {
FakeBroadcastChannel.channels.add(this);
}
postMessage(message: CrossTabSseMessage) {
if (FakeBroadcastChannel.throwOnTypes.has(message.type)) {
throw new Error(`postMessage failed for ${message.type}`);
}
if (FakeBroadcastChannel.muted) return;
const recipients = [...FakeBroadcastChannel.channels].filter(
(channel) => channel !== this && !channel.closed && channel.name === this.name,
);
queueMicrotask(() => {
for (const channel of recipients) {
if (channel.closed) continue;
channel.onmessage?.({ data: { ...message } });
}
});
}
static broadcastExternal(message: CrossTabSseMessage) {
queueMicrotask(() => {
for (const channel of FakeBroadcastChannel.channels) {
if (channel.closed) continue;
channel.onmessage?.({ data: { ...message } });
}
});
}
close() {
this.closed = true;
FakeBroadcastChannel.channels.delete(this);
}
static reset() {
for (const channel of FakeBroadcastChannel.channels) {
channel.closed = true;
}
FakeBroadcastChannel.channels.clear();
FakeBroadcastChannel.muted = false;
FakeBroadcastChannel.throwOnTypes.clear();
}
}
class Harness {
readonly sources: FakeEventSource[] = [];
readonly coordinators = new Map<string, CrossTabSseCoordinator>();
readonly visibility = new Map<string, TabVisibility>();
readonly visibilityHandlers = new Map<string, () => void>();
now = 1000;
createTab(tabId: string, visibility: TabVisibility = "visible") {
this.visibility.set(tabId, visibility);
const coordinator = createCrossTabSseCoordinator({
tabId,
channelFactory: (name) => new FakeBroadcastChannel(name),
eventSourceFactory: (url) => {
const source = new FakeEventSource(url, tabId);
this.sources.push(source);
return source;
},
getVisibility: () => this.visibility.get(tabId) ?? "visible",
addVisibilityChangeListener: (handler) => {
this.visibilityHandlers.set(tabId, handler);
return () => this.visibilityHandlers.delete(tabId);
},
addPagehideListener: () => () => {},
now: () => this.now,
timing: TEST_TIMING,
});
this.coordinators.set(tabId, coordinator);
return coordinator;
}
setVisibility(tabId: string, visibility: TabVisibility) {
this.visibility.set(tabId, visibility);
this.visibilityHandlers.get(tabId)?.();
}
openSources() {
return this.sources.filter((source) => !source.closed);
}
close() {
for (const coordinator of this.coordinators.values()) {
coordinator.close();
}
}
}
const harnesses: Harness[] = [];
afterEach(() => {
for (const harness of harnesses.splice(0)) {
harness.close();
}
FakeBroadcastChannel.reset();
});
describe("subscribeToCrossTabSse", () => {
test("opens one leader-owned global EventSource and keeps followers passive", async () => {
const harness = newHarness();
const cleanups = ["a", "b", "c"].map((tabId) => {
const coordinator = harness.createTab(tabId);
return subscribeForRunEvent(coordinator, []);
});
await waitFor(() => harness.openSources().length === 1);
expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]);
expect([...FakeBroadcastChannel.channels].every((channel) => channel.name === CROSS_TAB_SSE_CHANNEL)).toBe(true);
cleanups.forEach((cleanup) => cleanup());
});
test("leader broadcasts events to all local subscribers", async () => {
const harness = newHarness();
const keysByTab = new Map<string, string[]>();
for (const tabId of ["a", "b", "c"]) {
keysByTab.set(tabId, []);
subscribeForRunEvent(harness.createTab(tabId), keysByTab.get(tabId)!);
}
await waitFor(() => harness.openSources().length === 1);
clearRecordedKeys(keysByTab);
harness.openSources()[0].emit(runEvent({ id: "evt-1", runId: "run-1", seq: 1 }));
await waitFor(() => [...keysByTab.values()].every((keys) => keys.length === 1));
expect(keysByTab.get("a")).toEqual(["event"]);
expect(keysByTab.get("b")).toEqual(["event"]);
expect(keysByTab.get("c")).toEqual(["event"]);
});
test("board and run subscriptions coexist on the same global stream", async () => {
const harness = newHarness();
const coordinator = harness.createTab("a");
const boardKeys: string[] = [];
const runKeys: string[] = [];
subscribeForEvent(coordinator, {
subscriptionKey: "board",
keys: boardKeys,
resolveInvalidation: (payload) => ({
keys: payload.event === "run.running" ? ["board"] : [],
}),
resyncKeys: () => ["board-resync"],
});
subscribeForEvent(coordinator, {
subscriptionKey: "run:run-1",
keys: runKeys,
resolveInvalidation: (payload) => ({
keys: payload.event === "run.running" && payload.run_id === "run-1" ? ["run"] : [],
}),
resyncKeys: () => ["run-resync"],
});
await waitFor(() => harness.openSources().length === 1);
boardKeys.length = 0;
runKeys.length = 0;
harness.openSources()[0].emit(runEvent({ id: "evt-coexist", runId: "run-1", seq: 1 }));
expect(boardKeys).toEqual(["board"]);
expect(runKeys).toEqual(["run"]);
expect(harness.openSources().map((source) => source.url)).toEqual(["/api/v1/attach"]);
});
test("dedupes duplicate event ids until TTL or max-size eviction", async () => {
const harness = newHarness();
const keys: string[] = [];
subscribeForRunEvent(harness.createTab("a"), keys);
await waitFor(() => harness.openSources().length === 1);
keys.length = 0;
const source = harness.openSources()[0];
source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 }));
source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 }));
expect(keys).toEqual(["event"]);
harness.now += 5 * 60 * 1000 + 1;
source.emit(runEvent({ id: "evt-dup", runId: "run-1", seq: 1 }));
expect(keys).toEqual(["event", "event"]);
keys.length = 0;
for (let i = 0; i < 1001; i += 1) {
source.emit(runEvent({ id: `evt-${i}`, runId: "run-1", seq: i + 2 }));
}
source.emit(runEvent({ id: "evt-0", runId: "run-1", seq: 2 }));
expect(keys).toHaveLength(1002);
});
test("visible followers take over from a fresh hidden leader and resync", async () => {
const harness = newHarness();
const hiddenKeys: string[] = [];
const visibleKeys: string[] = [];
subscribeForRunEvent(harness.createTab("z", "hidden"), hiddenKeys);
await waitFor(() => harness.openSources().length === 1);
const hiddenSource = harness.openSources()[0];
subscribeForRunEvent(harness.createTab("a", "visible"), visibleKeys);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
expect(hiddenSource.closed).toBe(true);
expect(visibleKeys).toContain("resync");
});
test("visible candidates racing for the same hidden leader resolve lexically", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("b", "visible"), []);
subscribeForRunEvent(harness.createTab("a", "visible"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== "z");
expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]);
});
test("a lower lexical follower does not preempt a fresh visible leader", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "visible"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("a", "visible"), []);
await sleep(TEST_TIMING.electionJitterMs * 4);
expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]);
});
test("stale leader detection opens a new leader source and resyncs followers", async () => {
const harness = newHarness();
const followerKeys: string[] = [];
subscribeForRunEvent(harness.createTab("a"), []);
subscribeForRunEvent(harness.createTab("b"), followerKeys);
await waitFor(() => harness.openSources().length === 1);
const staleLeader = harness.openSources()[0];
harness.coordinators.get(staleLeader.owner)?.close();
harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1;
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner !== staleLeader.owner);
expect(followerKeys).toContain("resync");
});
test("simultaneous stale leader elections resolve to the lexical winner", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("b"), []);
subscribeForRunEvent(harness.createTab("a"), []);
await sleep(TEST_TIMING.heartbeatMs * 2);
harness.coordinators.get("z")?.close();
harness.now += TEST_TIMING.leaderStaleMs + TEST_TIMING.heartbeatMs + 1;
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
});
test("hidden leader ignores candidates for old observed leadership", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
const hiddenSource = harness.openSources()[0];
FakeBroadcastChannel.broadcastExternal({
type: "candidate",
version: 1,
tabId: "ghost",
sentAt: harness.now,
candidateId: "ghost",
candidateGeneration: 1,
visibility: "visible",
observedLeaderId: "z",
observedGeneration: 0,
reason: "hidden-leader",
});
await sleep(TEST_TIMING.electionJitterMs * 2);
expect(hiddenSource.closed).toBe(false);
expect(harness.openSources().map((source) => source.owner)).toEqual(["z"]);
});
test("prunes candidate records from older generations", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
const coordinator = harness.createTab("a", "visible");
subscribeForRunEvent(coordinator, []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
FakeBroadcastChannel.broadcastExternal({
type: "candidate",
version: 1,
tabId: "old-candidate",
sentAt: harness.now,
candidateId: "old-candidate",
candidateGeneration: 1,
visibility: "visible",
observedLeaderId: "previous-leader",
observedGeneration: 0,
reason: "stale-leader",
});
await sleep(TEST_TIMING.electionJitterMs * 2);
expect(candidateGenerations(coordinator)).not.toContain(1);
});
test("same-generation split brain converges to the higher-priority visible leader", async () => {
const harness = newHarness();
FakeBroadcastChannel.muted = true;
subscribeForRunEvent(harness.createTab("b"), []);
subscribeForRunEvent(harness.createTab("a"), []);
await waitFor(() => harness.openSources().length === 2);
FakeBroadcastChannel.muted = false;
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
});
test("old leader events are ignored after takeover", async () => {
const harness = newHarness();
const keys: string[] = [];
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1);
const oldSource = harness.openSources()[0];
subscribeForRunEvent(harness.createTab("a", "visible"), keys);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
keys.length = 0;
oldSource.emit(runEvent({ id: "evt-old", runId: "run-1", seq: 1 }));
expect(keys).toEqual([]);
});
test("old leader heartbeats are ignored after takeover", async () => {
const harness = newHarness();
subscribeForRunEvent(harness.createTab("z", "hidden"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "z");
subscribeForRunEvent(harness.createTab("a", "visible"), []);
await waitFor(() => harness.openSources().length === 1 && harness.openSources()[0].owner === "a");
FakeBroadcastChannel.broadcastExternal({
type: "heartbeat",
version: 1,
tabId: "z",
sentAt: harness.now,
leaderId: "z",
generation: 1,
visibility: "hidden",
});
await sleep(TEST_TIMING.heartbeatMs * 2);
expect(harness.openSources().map((source) => source.owner)).toEqual(["a"]);
});
test("last unsubscribe closes the leader source and releases leadership", async () => {
const harness = newHarness();
const cleanup = subscribeForRunEvent(harness.createTab("a"), []);
await waitFor(() => harness.openSources().length === 1);
const source = harness.openSources()[0];
cleanup();
expect(source.closed).toBe(true);
expect(harness.openSources()).toEqual([]);
});
test("missing BroadcastChannel uses subscriber fallback", () => {
const coordinator = createCrossTabSseCoordinator({
channelFactory: () => {
throw new Error("no channel");
},
});
let fallbackStarted = 0;
let fallbackStopped = 0;
const cleanup = subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey: "fallback",
mutate: (() => Promise.resolve()) as MutateFn,
resolveInvalidation: () => ({ keys: [] }),
resyncKeys: () => [],
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
debounceMs: 0,
});
cleanup();
expect(fallbackStarted).toBe(1);
expect(fallbackStopped).toBe(1);
});
test("postMessage failure after initialization degrades to fallback without coordinated resync", async () => {
const harness = newHarness();
const coordinator = harness.createTab("a");
const keys: string[] = [];
let fallbackStarted = 0;
let fallbackStopped = 0;
FakeBroadcastChannel.throwOnTypes.add("leader-changed");
const cleanup = subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey: "throwing-channel",
mutate: ((key: string) => {
keys.push(key);
return Promise.resolve();
}) as MutateFn,
resolveInvalidation: () => ({ keys: ["event"] }),
resyncKeys: () => ["resync"],
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
debounceMs: 0,
});
await waitFor(() => fallbackStarted === 1);
expect(harness.openSources()).toEqual([]);
expect(keys).toEqual([]);
cleanup();
expect(fallbackStopped).toBe(1);
});
test("close resets coordination availability after an initial channel failure", async () => {
let channelUnavailable = true;
const sources: FakeEventSource[] = [];
const coordinator = createCrossTabSseCoordinator({
tabId: "recovering",
channelFactory: (name) => {
if (channelUnavailable) throw new Error("channel unavailable");
return new FakeBroadcastChannel(name);
},
eventSourceFactory: (url) => {
const source = new FakeEventSource(url, "recovering");
sources.push(source);
return source;
},
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: TEST_TIMING,
});
let firstFallbackStarted = 0;
let secondFallbackStarted = 0;
const firstCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
firstFallbackStarted += 1;
return () => {};
},
});
firstCleanup();
coordinator.close();
channelUnavailable = false;
const secondCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
secondFallbackStarted += 1;
return () => {};
},
});
await waitFor(() => sources.some((source) => !source.closed));
expect(firstFallbackStarted).toBe(1);
expect(secondFallbackStarted).toBe(0);
expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]);
secondCleanup();
coordinator.close();
});
test("last unsubscribe retries coordination after an initial channel failure", async () => {
let channelUnavailable = true;
const sources: FakeEventSource[] = [];
const coordinator = createCrossTabSseCoordinator({
tabId: "retry-after-unsubscribe",
channelFactory: (name) => {
if (channelUnavailable) throw new Error("channel unavailable");
return new FakeBroadcastChannel(name);
},
eventSourceFactory: (url) => {
const source = new FakeEventSource(url, "retry-after-unsubscribe");
sources.push(source);
return source;
},
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: TEST_TIMING,
});
let fallbackStarted = 0;
const firstCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {};
},
});
firstCleanup();
channelUnavailable = false;
const secondCleanup = subscribeWithFallback(coordinator, {
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {};
},
});
await waitFor(() => sources.some((source) => !source.closed));
expect(fallbackStarted).toBe(1);
expect(sources.filter((source) => !source.closed).map((source) => source.url)).toEqual(["/api/v1/attach"]);
secondCleanup();
coordinator.close();
});
test("close stops fallback subscriptions added after degradation", async () => {
const harness = newHarness();
const coordinator = harness.createTab("a");
let fallbackStarted = 0;
let fallbackStopped = 0;
FakeBroadcastChannel.throwOnTypes.add("leader-changed");
subscribeWithFallback(coordinator, {
subscriptionKey: "before-degrade",
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
});
await waitFor(() => fallbackStarted === 1);
FakeBroadcastChannel.throwOnTypes.clear();
subscribeWithFallback(coordinator, {
subscriptionKey: "after-degrade",
fallbackSubscribe: () => {
fallbackStarted += 1;
return () => {
fallbackStopped += 1;
};
},
});
expect(fallbackStarted).toBe(2);
coordinator.close();
expect(fallbackStopped).toBe(2);
});
});
function newHarness() {
const harness = new Harness();
harnesses.push(harness);
return harness;
}
function subscribeForRunEvent(coordinator: CrossTabSseCoordinator, keys: string[]) {
return subscribeForEvent(coordinator, {
subscriptionKey: "run-feed",
keys,
resolveInvalidation: (payload) => ({
keys: payload.event === "run.running" ? ["event"] : [],
}),
resyncKeys: () => ["resync"],
});
}
function subscribeForEvent(
coordinator: CrossTabSseCoordinator,
{
subscriptionKey,
keys,
resolveInvalidation,
resyncKeys,
}: {
subscriptionKey: string;
keys: string[];
resolveInvalidation: (payload: EventPayload) => { keys: string[] };
resyncKeys: () => string[];
},
) {
return subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey,
mutate: ((key: string) => {
keys.push(key);
return Promise.resolve();
}) as MutateFn,
resolveInvalidation,
resyncKeys,
fallbackSubscribe: () => {
throw new Error("fallback should not be used");
},
debounceMs: 0,
});
}
function subscribeWithFallback(
coordinator: CrossTabSseCoordinator,
{
subscriptionKey = "fallback-test",
fallbackSubscribe,
}: {
subscriptionKey?: string;
fallbackSubscribe: () => () => void;
},
) {
return subscribeToCrossTabSse<EventPayload>({
coordinator,
subscriptionKey,
mutate: (() => Promise.resolve()) as MutateFn,
resolveInvalidation: () => ({ keys: [] }),
resyncKeys: () => [],
fallbackSubscribe,
debounceMs: 0,
});
}
function candidateGenerations(coordinator: CrossTabSseCoordinator): number[] {
const inspectable = coordinator as unknown as {
candidates: Map<string, { candidateGeneration: number }>;
};
return [...inspectable.candidates.values()].map((candidate) => candidate.candidateGeneration);
}
function runEvent({
id,
runId,
seq,
}: {
id: string;
runId: string;
seq: number;
}) {
return {
id,
seq,
run_id: runId,
event: "run.running",
ts: "2026-05-04T12:00:00.000Z",
};
}
async function waitFor(condition: () => boolean, timeoutMs = 500) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (condition()) return;
await sleep(2);
}
throw new Error("condition did not become true before timeout");
}
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function clearRecordedKeys(keysByTab: Map<string, string[]>) {
for (const keys of keysByTab.values()) {
keys.length = 0;
}
}

1175
apps/fabro-web/app/lib/cross-tab-sse.ts Normal file
View file

File diff suppressed because it is too large Load diff

34
apps/fabro-web/app/lib/queries.ts
View file

@ -1,12 +1,11 @@
import useSWR, { type SWRConfiguration } from "swr";
import type {
ApiQuestion,
EventEnvelope,
PaginatedBoardRunList,
PaginatedEventList,
PaginatedRunFileList,
PaginatedRunList,
PaginatedRunStageList,
PaginatedStageTurnList,
CommandLogResponse,
CommandOutputStream,
RunBilling,
@ -24,6 +23,7 @@ import {
apiNullableTextFetcher,
apiPaginatedFetcher,
apiTextFetcher,
fetchAllStageEvents,
type PaginatedEnvelope,
} from "./api-client";
import { queryKeys } from "./query-keys";
@ -34,6 +34,9 @@ const immutableOptions: SWRConfiguration = {
revalidateOnReconnect: false,
};
type BoardRunsEnvelope = PaginatedEnvelope<PaginatedBoardRunList["data"][number]> &
Pick<PaginatedBoardRunList, "columns">;
export function useAuthConfig() {
return useSWR<{ methods: string[] }>(queryKeys.auth.config(), apiFetcher, immutableOptions);
}
@ -61,11 +64,7 @@ export function useSystemInfo() {
}
export function useBoardsRuns() {
return useSWR<
PaginatedEnvelope<PaginatedBoardRunList["data"][number]> & {
columns: { id: string; name: string }[];
}
>(queryKeys.boards.runs(), apiPaginatedFetcher);
return useSWR<BoardRunsEnvelope>(queryKeys.boards.runs(), apiPaginatedFetcher);
}
export function useRun(id: string | undefined) {
@ -141,21 +140,10 @@ export function useRunQuestions(id: string | undefined, enabled: boolean) {
);
}
export function useRunStageTurns(
id: string | undefined,
stageId: string | undefined,
enabled = true,
) {
return useSWR<PaginatedStageTurnList | null>(
id && stageId && enabled ? queryKeys.runs.stageTurns(id, stageId) : null,
apiNullableFetcher,
);
}
export function useRunEventsList(id: string | undefined, enabled = true) {
return useSWR<PaginatedEventList | null>(
id && enabled ? queryKeys.runs.events(id, 1000) : null,
apiNullableFetcher,
export function useRunStageEvents(id: string | undefined, stageId: string | undefined) {
return useSWR<EventEnvelope[]>(
id && stageId ? queryKeys.runs.stageEvents(id, stageId) : null,
fetchAllStageEvents<EventEnvelope>,
);
}
@ -206,4 +194,4 @@ export function useServerSettings() {
return useSWR<ServerSettings>(queryKeys.settings.server(), apiFetcher, immutableOptions);
}
export { apiTextFetcher };
export { apiTextFetcher };

25
apps/fabro-web/app/lib/query-keys.test.ts
View file

@ -11,6 +11,9 @@ describe("queryKeys", () => {
expect(queryKeys.runs.stageLog("run 1", "build step@2", "stderr", 12, 34)).toBe(
"/api/v1/runs/run%201/stages/build%20step%402/logs/stderr?offset=12&limit=34",
);
expect(queryKeys.runs.stageEvents("run 1", "build step", 7, 25)).toBe(
"/api/v1/runs/run%201/stages/build%20step/events?since_seq=7&limit=25",
);
});
test("event-mapped keys match query hook resources", () => {
@ -19,11 +22,31 @@ describe("queryKeys", () => {
]);
expect(queryKeysForRunEvent("run-1", "stage.completed", "stage-1")).toEqual([
queryKeys.runs.stages("run-1"),
queryKeys.runs.billing("run-1"),
queryKeys.runs.events("run-1", 1000),
queryKeys.runs.graph("run-1", "LR"),
queryKeys.runs.graph("run-1", "TB"),
queryKeys.runs.detail("run-1"),
queryKeys.runs.stageTurns("run-1", "stage-1"),
queryKeys.runs.stageEvents("run-1", "stage-1"),
]);
});
test("agent activity events invalidate the per-stage events key", () => {
for (const event of [
"stage.prompt",
"agent.message",
"agent.tool.started",
"agent.tool.completed",
"command.started",
"command.completed",
]) {
expect(queryKeysForRunEvent("run-1", event, "stage-1")).toEqual([
queryKeys.runs.stageEvents("run-1", "stage-1"),
]);
}
});
test("agent activity events without a node_id invalidate nothing", () => {
expect(queryKeysForRunEvent("run-1", "agent.message")).toEqual([]);
});
});

9
apps/fabro-web/app/lib/query-keys.ts
View file

@ -44,8 +44,11 @@ export const queryKeys = {
}),
events: (id: string, limit = 1000) =>
withQuery(`/api/v1/runs/${pathSegment(id)}/events`, { limit }),
stageTurns: (id: string, stageId: string) =>
`/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/turns`,
stageEvents: (id: string, stageId: string, sinceSeq?: number, limit?: number) =>
withQuery(`/api/v1/runs/${pathSegment(id)}/stages/${pathSegment(stageId)}/events`, {
since_seq: sinceSeq,
limit,
}),
stageLog: (
id: string,
stageId: string,
@ -75,4 +78,4 @@ export const queryKeys = {
settings: {
server: () => "/api/v1/settings",
},
};
};

221
apps/fabro-web/app/lib/run-events.test.tsx
View file

@ -4,7 +4,12 @@ import {
queryKeysForRunEvent,
subscribeToRunEvents,
} from "./run-events";
import {
createCrossTabSseCoordinator,
type BroadcastChannelLike,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import type { EventSourceLike } from "./sse";
type MessageHandler = ((event: { data: string }) => void) | null;
@ -25,6 +30,14 @@ class FakeEventSource {
}
}
class FakeBroadcastChannel implements BroadcastChannelLike {
onmessage: ((event: { data: unknown }) => void) | null = null;
postMessage() {}
close() {}
}
describe("queryKeysForRunEvent", () => {
test("terminal events invalidate run-scoped resources", () => {
expect(queryKeysForRunEvent("run-1", "run.completed")).toEqual([
@ -36,13 +49,98 @@ describe("queryKeysForRunEvent", () => {
queryKeys.runs.graph("run-1", "TB"),
]);
});
test("stage.retrying invalidates stages, billing, events, graph, detail, and stage events", () => {
expect(queryKeysForRunEvent("run-1", "stage.retrying", "verify@2")).toEqual([
queryKeys.runs.stages("run-1"),
queryKeys.runs.billing("run-1"),
queryKeys.runs.events("run-1", 1000),
queryKeys.runs.graph("run-1", "LR"),
queryKeys.runs.graph("run-1", "TB"),
queryKeys.runs.detail("run-1"),
queryKeys.runs.stageEvents("run-1", "verify@2"),
]);
});
test("stage-scoped steering events invalidate run events and stage events", () => {
expect(queryKeysForRunEvent("run-1", "agent.steering.injected", "agent@1")).toEqual([
queryKeys.runs.events("run-1", 1000),
queryKeys.runs.stageEvents("run-1", "agent@1"),
]);
});
});
describe("subscribeToRunEvents", () => {
test("refcounts shared sources and keeps mutators active until final unsubscribe", () => {
test("coordinated mode uses the global attach stream and filters by run_id", async () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createCoordinator((url) => {
created.push(url);
return source;
});
const cleanup = subscribeToRunEvents(
"run-coordinated",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => {
throw new Error("source should be created by coordinator");
},
{ debounceMs: 0, coordinator },
);
await waitFor(() => created.length === 1);
keys.length = 0;
source.emit({ event: "checkpoint.completed", run_id: "other-run" });
source.emit({ event: "checkpoint.completed", run_id: "run-coordinated" });
expect(created).toEqual(["/api/v1/attach"]);
expect(keys).toEqual([queryKeys.runs.files("run-coordinated")]);
cleanup();
coordinator.close();
});
test("coordinated terminal events invalidate without closing the global stream", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-terminal",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
keys.length = 0;
source.emit({ event: "run.failed", run_id: "run-terminal" });
expect(source.closed).toBe(false);
expect(keys).toContain(queryKeys.runs.files("run-terminal"));
expect(keys).toContain(queryKeys.runs.billing("run-terminal"));
keys.length = 0;
source.emit({ event: "run.archived", run_id: "run-terminal" });
expect(source.closed).toBe(false);
expect(keys).toEqual([queryKeys.runs.detail("run-terminal")]);
cleanup();
coordinator.close();
});
test("fallback refcounts run-scoped sources and keeps mutators active until final unsubscribe", () => {
const source = new FakeEventSource();
const created: string[] = [];
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const mutate = (key: string) => {
keys.push(key);
return Promise.resolve();
@ -51,10 +149,10 @@ describe("subscribeToRunEvents", () => {
const firstCleanup = subscribeToRunEvents("run-refcount", mutate, (url) => {
created.push(url);
return source;
}, { debounceMs: 0 });
}, { debounceMs: 0, coordinator });
const secondCleanup = subscribeToRunEvents("run-refcount", mutate, () => {
throw new Error("source should be reused");
}, { debounceMs: 0 });
}, { debounceMs: 0, coordinator });
expect(created).toEqual(["/api/v1/runs/run-refcount/attach"]);
@ -66,22 +164,29 @@ describe("subscribeToRunEvents", () => {
secondCleanup();
expect(source.closed).toBe(true);
coordinator.close();
});
test("runs payload callbacks for later subscribers on a shared source", () => {
test("fallback runs payload callbacks for later subscribers on a shared source", () => {
const source = new FakeEventSource();
const seen: string[] = [];
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const mutate = (key: string) => {
keys.push(key);
return Promise.resolve();
};
const callbackMutate = () => Promise.resolve();
const firstCleanup = subscribeToRunEvents("run-shared-payload", mutate, () => source, { debounceMs: 0 });
const secondCleanup = subscribeToRunEvents("run-shared-payload", mutate, () => {
const firstCleanup = subscribeToRunEvents("run-shared-payload", mutate, () => source, {
debounceMs: 0,
coordinator,
});
const secondCleanup = subscribeToRunEvents("run-shared-payload", callbackMutate, () => {
throw new Error("source should be reused");
}, {
debounceMs: 0,
coordinator,
onEvent: (payload) => {
if (payload.event) seen.push(payload.event);
},
@ -94,11 +199,13 @@ describe("subscribeToRunEvents", () => {
firstCleanup();
secondCleanup();
coordinator.close();
});
test("terminal events close the source after invalidating keys", () => {
test("fallback terminal events close the source after invalidating keys", () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const cleanup = subscribeToRunEvents(
"run-terminal",
(key) => {
@ -106,7 +213,7 @@ describe("subscribeToRunEvents", () => {
return Promise.resolve();
},
() => source,
{ debounceMs: 0 },
{ debounceMs: 0, coordinator },
);
source.emit({ event: "run.failed" });
@ -116,13 +223,72 @@ describe("subscribeToRunEvents", () => {
expect(keys).toContain(queryKeys.runs.billing("run-terminal"));
cleanup();
coordinator.close();
});
test("malformed events are ignored and StrictMode-style cleanup does not underflow", () => {
test("envelope with suffixed stage_id invalidates stageEvents(runId, stageId)", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-stage",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
source.emit({
event: "stage.retrying",
run_id: "run-stage",
stage_id: "verify@2",
node_id: "verify",
});
expect(keys).toContain(queryKeys.runs.stageEvents("run-stage", "verify@2"));
expect(keys).toContain(queryKeys.runs.stages("run-stage"));
expect(keys).toContain(queryKeys.runs.events("run-stage", 1000));
expect(keys).toContain(queryKeys.runs.graph("run-stage", "LR"));
expect(keys).toContain(queryKeys.runs.detail("run-stage"));
expect(keys).not.toContain(queryKeys.runs.stageEvents("run-stage", "verify"));
cleanup();
coordinator.close();
});
test("falls back to node_id when an event has no stage_id", async () => {
const source = new FakeEventSource();
const keys: string[] = [];
const coordinator = createCoordinator(() => source);
const cleanup = subscribeToRunEvents(
"run-stage-node",
(key) => {
keys.push(key);
return Promise.resolve();
},
() => source,
{ debounceMs: 0, coordinator },
);
await waitFor(() => source.onmessage !== null);
source.emit({ event: "stage.started", run_id: "run-stage-node", node_id: "verify" });
expect(keys).toContain(queryKeys.runs.stageEvents("run-stage-node", "verify"));
expect(keys).toContain(queryKeys.runs.stages("run-stage-node"));
cleanup();
coordinator.close();
});
test("fallback malformed events are ignored and StrictMode-style cleanup does not underflow", () => {
const firstSource = new FakeEventSource();
const secondSource = new FakeEventSource();
const sources = [firstSource, secondSource];
const keys: string[] = [];
const coordinator = createFallbackCoordinator();
const firstCleanup = subscribeToRunEvents(
"run-strict",
@ -131,7 +297,7 @@ describe("subscribeToRunEvents", () => {
return Promise.resolve();
},
() => sources.shift()!,
{ debounceMs: 0 },
{ debounceMs: 0, coordinator },
);
firstSource.emitRaw("{broken");
firstCleanup();
@ -143,12 +309,45 @@ describe("subscribeToRunEvents", () => {
return Promise.resolve();
},
() => sources.shift()!,
{ debounceMs: 0 },
{ debounceMs: 0, coordinator },
);
secondCleanup();
expect(keys).toEqual([]);
expect(firstSource.closed).toBe(true);
expect(secondSource.closed).toBe(true);
coordinator.close();
});
});
function createCoordinator(eventSourceFactory: (url: string) => EventSourceLike) {
return createCrossTabSseCoordinator({
tabId: "run-test",
channelFactory: () => new FakeBroadcastChannel(),
eventSourceFactory,
addVisibilityChangeListener: () => () => {},
addPagehideListener: () => () => {},
timing: {
heartbeatMs: 10,
leaderStaleMs: 50,
electionJitterMs: 0,
},
});
}
function createFallbackCoordinator() {
return createCrossTabSseCoordinator({
channelFactory: () => {
throw new Error("BroadcastChannel unavailable");
},
});
}
async function waitFor(condition: () => boolean, timeoutMs = 200) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (condition()) return;
await new Promise((resolve) => setTimeout(resolve, 2));
}
throw new Error("condition did not become true before timeout");
}

124
apps/fabro-web/app/lib/run-events.ts
View file

@ -1,6 +1,10 @@
import { useEffect } from "react";
import { useSWRConfig } from "swr";
import {
subscribeToCrossTabSse,
type CrossTabSseCoordinator,
} from "./cross-tab-sse";
import { queryKeys } from "./query-keys";
import {
createBrowserEventSource,
@ -15,10 +19,18 @@ export interface RunEventPayload extends EventPayload {
id?: string;
seq?: number;
event?: string;
run_id?: string;
node_id?: string;
stage_id?: string;
properties?: Record<string, unknown>;
}
interface RunEventOptions {
debounceMs?: number;
coordinator?: CrossTabSseCoordinator;
onEvent?: (payload: RunEventPayload) => void;
}
const subscriptions = new Map<string, SharedEventSubscription>();
const TERMINAL_EVENTS = new Set(["run.completed", "run.failed"]);
@ -34,8 +46,31 @@ const RUN_SUMMARY_EVENTS = new Set([
"run.archived",
"run.unarchived",
]);
const STAGE_EVENTS = new Set(["stage.started", "stage.completed", "stage.failed"]);
const COMMAND_EVENTS = new Set(["command.started", "command.completed"]);
const STAGE_EVENTS = new Set([
"stage.started",
"stage.completed",
"stage.failed",
"stage.retrying",
]);
// Single source of truth: every event type the `eventsToActivity` reducer in
// `routes/run-stages.tsx` consumes. When any of these arrive for a stage we
// currently view, the stage-events SWR key for that stage must be invalidated
// so the panel refetches. The reducer imports this list so the switch stays
// in sync with the invalidation set; if the reducer grows a new case, this
// list is the single edit point.
//
// The lifecycle `STAGE_EVENTS` set is kept separate because it also fans out
// to run-scoped invalidations (stages list, graph, detail).
export const STAGE_ACTIVITY_EVENT_TYPES = [
"stage.prompt",
"agent.message",
"agent.tool.started",
"agent.tool.completed",
"command.started",
"command.completed",
] as const;
export type StageActivityEventType = (typeof STAGE_ACTIVITY_EVENT_TYPES)[number];
const STAGE_ACTIVITY_EVENTS = new Set<string>(STAGE_ACTIVITY_EVENT_TYPES);
const INTERVIEW_EVENTS = new Set([
"interview.started",
"interview.completed",
@ -84,32 +119,26 @@ export function queryKeysForRunEvent(
if (STAGE_EVENTS.has(event)) {
const keys = [
queryKeys.runs.stages(runId),
queryKeys.runs.billing(runId),
queryKeys.runs.events(runId, 1000),
queryKeys.runs.graph(runId, "LR"),
queryKeys.runs.graph(runId, "TB"),
queryKeys.runs.detail(runId),
];
if (stageId) {
keys.push(queryKeys.runs.stageTurns(runId, stageId));
keys.push(queryKeys.runs.stageEvents(runId, stageId));
}
return keys;
}
if (COMMAND_EVENTS.has(event)) {
const keys = [
queryKeys.runs.stages(runId),
queryKeys.runs.events(runId, 1000),
];
if (stageId) {
keys.push(queryKeys.runs.stageTurns(runId, stageId));
}
return keys;
if (STAGE_ACTIVITY_EVENTS.has(event)) {
return stageId ? [queryKeys.runs.stageEvents(runId, stageId)] : [];
}
if (STEERING_EVENTS.has(event)) {
const keys = [queryKeys.runs.events(runId, 1000)];
if (stageId) {
keys.push(queryKeys.runs.stageTurns(runId, stageId));
keys.push(queryKeys.runs.stageEvents(runId, stageId));
}
return keys;
}
@ -121,40 +150,61 @@ export function subscribeToRunEvents(
runId: string,
mutate: MutateFn,
eventSourceFactory: (url: string) => EventSourceLike = createBrowserEventSource,
{
debounceMs = 300,
onEvent,
}: {
debounceMs?: number;
onEvent?: (payload: RunEventPayload) => void;
} = {},
{ debounceMs = 300, coordinator, onEvent }: RunEventOptions = {},
): () => void {
return subscribeToSharedEventSource<RunEventPayload>({
subscriptions,
subscriptionKey: runId,
url: queryKeys.runs.attach(runId),
return subscribeToCrossTabSse<RunEventPayload>({
coordinator,
subscriptionKey: `run:${runId}`,
mutate,
eventSourceFactory,
debounceMs,
resyncKeys: () => resyncKeysForRun(runId),
resolveInvalidation: (payload) => {
if (payload.run_id !== runId) return { keys: [] };
onEvent?.(payload);
const event = payload.event;
if (!event) return { keys: [] };
const stageId = stageIdFromPayload(payload);
const keys = queryKeysForRunEvent(runId, event, stageId);
const terminal = TERMINAL_EVENTS.has(event);
return {
keys,
close: terminal,
immediate: terminal,
};
return runInvalidation(runId, payload);
},
fallbackSubscribe: () =>
subscribeToSharedEventSource<RunEventPayload>({
subscriptions,
subscriptionKey: runId,
url: queryKeys.runs.attach(runId),
mutate,
eventSourceFactory,
debounceMs,
resolveInvalidation: (payload) => {
onEvent?.(payload);
const result = runInvalidation(runId, payload);
return { ...result, close: result.immediate };
},
}),
});
}
function runInvalidation(runId: string, payload: RunEventPayload) {
const event = payload.event;
if (!event) return { keys: [], immediate: false };
const stageId = stageIdFromPayload(payload);
const keys = queryKeysForRunEvent(runId, event, stageId);
const terminal = TERMINAL_EVENTS.has(event);
return { keys, immediate: terminal };
}
function resyncKeysForRun(runId: string) {
return [
queryKeys.runs.detail(runId),
queryKeys.runs.files(runId),
queryKeys.runs.billing(runId),
queryKeys.runs.stages(runId),
queryKeys.runs.events(runId, 1000),
queryKeys.runs.graph(runId, "LR"),
queryKeys.runs.graph(runId, "TB"),
queryKeys.runs.questions(runId, 25, 0),
];
}
function stageIdFromPayload(payload: RunEventPayload): string | undefined {
if (typeof payload.stage_id === "string") return payload.stage_id;
if (typeof payload.node_id === "string") return payload.node_id;
const nodeId = payload.properties?.node_id;
return typeof nodeId === "string" ? nodeId : undefined;

172
apps/fabro-web/app/lib/stage-sidebar.test.ts Normal file
View file

@ -0,0 +1,172 @@
import { describe, expect, test } from "bun:test";
import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client";
import type { Stage } from "../components/stage-sidebar";
import { aggregateGraphNodeStatus, formatStageLabel, mapRunStagesToSidebarStages } from "./stage-sidebar";
function makeStage(nodeId: string, visit: number, status: StageState): Stage {
return {
id: `${nodeId}@${visit}`,
name: nodeId,
nodeId,
visit,
status,
duration: "--",
};
}
describe("mapRunStagesToSidebarStages", () => {
test("maps two visits of the same node to distinct sidebar entries", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "apply-changes@1",
name: "Apply Changes",
status: "succeeded",
duration_secs: 12.5,
node_id: "apply",
visit: 1,
},
{
id: "apply-changes@2",
name: "Apply Changes",
status: "running",
node_id: "apply",
visit: 2,
},
],
meta: { has_more: false },
};
const result = mapRunStagesToSidebarStages(stages);
expect(result).toHaveLength(2);
expect(result[0].id).toBe("apply-changes@1");
expect(result[0].nodeId).toBe("apply");
expect(result[0].visit).toBe(1);
expect(formatStageLabel(result[0])).toBe("Apply Changes");
expect(result[1].id).toBe("apply-changes@2");
expect(result[1].nodeId).toBe("apply");
expect(result[1].visit).toBe(2);
expect(formatStageLabel(result[1])).toBe("Apply Changes (2)");
});
test("filters by node_id (suffixed start@1 / exit@1 are still hidden)", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "start@1",
name: "start",
status: "succeeded",
node_id: "start",
visit: 1,
},
{
id: "verify@1",
name: "verify",
status: "succeeded",
node_id: "verify",
visit: 1,
},
{
id: "exit@1",
name: "exit",
status: "succeeded",
node_id: "exit",
visit: 1,
},
],
meta: { has_more: false },
};
const result = mapRunStagesToSidebarStages(stages);
expect(result.map((s) => s.id)).toEqual(["verify@1"]);
});
test("missing duration renders as '--'", () => {
const stages: PaginatedRunStageList = {
data: [
{
id: "verify@1",
name: "verify",
status: "running",
node_id: "verify",
visit: 1,
},
],
meta: { has_more: false },
};
expect(mapRunStagesToSidebarStages(stages)[0].duration).toBe("--");
});
});
describe("aggregateGraphNodeStatus", () => {
test("(failed, running) renders as running and clicks open the latest visit", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "failed"),
makeStage("verify", 2, "running"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "running",
latestStageId: "verify@2",
});
});
test("(failed, succeeded) renders as succeeded — failure-then-fix shows healed", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "failed"),
makeStage("verify", 2, "succeeded"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "succeeded",
latestStageId: "verify@2",
});
});
test("(succeeded, failed) renders as failed and clicks open the latest visit", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "succeeded"),
makeStage("verify", 2, "failed"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "failed",
latestStageId: "verify@2",
});
});
test("(running, retrying) — latest active wins", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 1, "running"),
makeStage("verify", 2, "retrying"),
]);
expect(result.get("verify")).toEqual({
displayStatus: "retrying",
latestStageId: "verify@2",
});
});
test("orders by visit even when input is shuffled", () => {
const result = aggregateGraphNodeStatus([
makeStage("verify", 2, "running"),
makeStage("verify", 1, "failed"),
]);
expect(result.get("verify")?.latestStageId).toBe("verify@2");
});
test("single visit per node is unaffected", () => {
const result = aggregateGraphNodeStatus([
makeStage("plan", 1, "succeeded"),
makeStage("apply", 1, "running"),
]);
expect(result.get("plan")).toEqual({
displayStatus: "succeeded",
latestStageId: "plan@1",
});
expect(result.get("apply")).toEqual({
displayStatus: "running",
latestStageId: "apply@1",
});
});
});

70
apps/fabro-web/app/lib/stage-sidebar.ts
View file

@ -1,27 +1,83 @@
import type { PaginatedRunStageList, StageState } from "@qltysh/fabro-api-client";
import { StageState } from "@qltysh/fabro-api-client";
import type { PaginatedRunStageList } from "@qltysh/fabro-api-client";
import type { Stage } from "../components/stage-sidebar";
import { isVisibleStage } from "../data/runs";
import { formatDurationSecs } from "./format";
export const ACTIVE_STAGE_STATES: ReadonlySet<StageState> = new Set(["running", "retrying"]);
export const SUCCEEDED_STAGE_STATES: ReadonlySet<StageState> = new Set([
"succeeded",
"partially_succeeded",
export const ACTIVE_STAGE_STATES: ReadonlySet<StageState> = new Set([
StageState.RUNNING,
StageState.RETRYING,
]);
export const IN_FLIGHT_STAGE_STATES: ReadonlySet<StageState> = new Set([
StageState.PENDING,
StageState.RUNNING,
StageState.RETRYING,
]);
export const SUCCEEDED_STAGE_STATES: ReadonlySet<StageState> = new Set([
StageState.SUCCEEDED,
StageState.PARTIALLY_SUCCEEDED,
]);
/**
* Display label for a stage. Suffixes `(N)` for visits > 1 so a looped node
* (e.g. `verify`) renders as `verify`, `verify (2)`, `verify (3)` in the
* sidebar and stage header.
*/
export function formatStageLabel(stage: { name: string; visit: number }): string {
return stage.visit > 1 ? `${stage.name} (${stage.visit})` : stage.name;
}
export function mapRunStagesToSidebarStages(
stagesResult: PaginatedRunStageList | null | undefined,
): Stage[] {
return (stagesResult?.data ?? [])
.filter((stage) => isVisibleStage(stage.id))
.filter((stage) => isVisibleStage(stage.node_id))
.map((stage) => ({
id: stage.id,
name: stage.name,
dotId: stage.dot_id ?? stage.id,
nodeId: stage.node_id,
visit: stage.visit,
status: stage.status,
duration: stage.duration_secs != null
? formatDurationSecs(stage.duration_secs)
: "--",
}));
}
/**
* Aggregate per-node display state for the workflow graph.
*
* Status policy: if any visit is active (running/retrying), the node renders
* that active state (latest active visit wins). Otherwise the node renders
* the latest visit's terminal state. The click target is always the latest
* visit's stageId.
*/
export function aggregateGraphNodeStatus(stages: readonly Stage[]): Map<
string,
{ displayStatus: StageState; latestStageId: string }
> {
// Single pass per nodeId: track the visit with the highest `visit` overall
// (drives click target + terminal status) and the highest-visit *active*
// stage (drives display when any visit is in flight).
const latest = new Map<string, Stage>();
const latestActive = new Map<string, Stage>();
for (const stage of stages) {
const prevLatest = latest.get(stage.nodeId);
if (!prevLatest || stage.visit > prevLatest.visit) {
latest.set(stage.nodeId, stage);
}
if (ACTIVE_STAGE_STATES.has(stage.status)) {
const prevActive = latestActive.get(stage.nodeId);
if (!prevActive || stage.visit > prevActive.visit) {
latestActive.set(stage.nodeId, stage);
}
}
}
const result = new Map<string, { displayStatus: StageState; latestStageId: string }>();
for (const [nodeId, latestStage] of latest) {
const display = latestActive.get(nodeId) ?? latestStage;
result.set(nodeId, { displayStatus: display.status, latestStageId: latestStage.id });
}
return result;
}

19
apps/fabro-web/app/lib/time.ts
View file

@ -1,3 +1,21 @@
import { useEffect, useState } from "react";
/**
* Re-renders the calling component every `intervalMs` milliseconds while
* `active` is true, returning the current `Date.now()` value at each tick.
* Returns the captured value when paused, so renders are stable.
*/
export function useTickingNow(active: boolean, intervalMs = 1000): number {
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!active) return;
setNow(Date.now());
const interval = setInterval(() => setNow(Date.now()), intervalMs);
return () => clearInterval(interval);
}, [active, intervalMs]);
return now;
}
function relativeTime(seconds: number, past: boolean): string {
if (seconds < 60) return past ? "just now" : "in <1m";
const minutes = Math.floor(seconds / 60);
@ -21,4 +39,3 @@ export function timeAgo(iso: string): string {
export function timeUntil(iso: string): string {
return relativeTime(Math.floor((new Date(iso).getTime() - Date.now()) / 1000), false);
}

62
apps/fabro-web/app/routes/run-billing.test.tsx
View file

@ -75,12 +75,14 @@ describe("RunBilling", () => {
model: null,
billing: zeroBilling(),
runtime_secs: 0,
state: "succeeded",
},
{
stage: { id: "command", name: "command" },
model: null,
billing: zeroBilling(),
runtime_secs: 61,
state: "succeeded",
},
],
totals: {
@ -96,7 +98,7 @@ describe("RunBilling", () => {
expect(text).toMatch(/—\s*\/\s*—/);
expect(text).toContain("1m 1s");
expect(text).not.toContain("By model");
expect(text).not.toContain("No completed stages yet");
expect(text).not.toContain("No stages yet");
});
test("renders mixed LLM and non-LLM rows while counting only LLM rows by model", () => {
@ -108,6 +110,7 @@ describe("RunBilling", () => {
model: null,
billing: zeroBilling(),
runtime_secs: 0,
state: "succeeded",
},
{
stage: { id: "agent", name: "agent" },
@ -119,6 +122,7 @@ describe("RunBilling", () => {
total_usd_micros: 240000,
}),
runtime_secs: 42,
state: "succeeded",
},
],
totals: {
@ -155,11 +159,61 @@ describe("RunBilling", () => {
expect(textFromInstance(byModelFooterCells[1])).toBe("1");
});
test("keeps the empty state for runs with no completed stages", () => {
test("keeps the empty state for runs with no stages", () => {
const renderer = renderBilling(billing());
const text = textFromNode(renderer.toJSON());
expect(text).toContain("No completed stages yet");
expect(text).toContain("Stages will appear once the run produces completed nodes.");
expect(text).toContain("No stages yet");
expect(text).toContain("Stages will appear as soon as the run starts executing.");
});
test("renders an in-flight row with live runtime and includes its elapsed time in the footer", () => {
const originalNow = Date.now;
// Pin "now" to 30s after the in-flight row started.
const startedAt = "2026-04-29T12:00:00.000Z";
const fakeNow = new Date("2026-04-29T12:00:30.000Z").getTime();
Date.now = () => fakeNow;
try {
const renderer = renderBilling(
billing({
stages: [
{
stage: { id: "in-flight", name: "in-flight" },
model: null,
// Server reports 0 runtime / no billing; the row is still being executed.
billing: zeroBilling(),
runtime_secs: 0,
started_at: startedAt,
state: "running",
},
],
// Server total is 0 because the in-flight row hasn't been finalized.
totals: {
runtime_secs: 0,
...zeroBilling(),
},
}),
);
const text = textFromNode(renderer.toJSON());
// Empty-state must NOT show — the table should appear as soon as the
// first stage starts.
expect(text).not.toContain("No stages yet");
expect(text).toContain("in-flight");
// Both the row's runtime cell and the footer total should reflect
// ~30s elapsed since started_at.
expect(text).toContain("30s");
const footers = renderer.root.findAll((node) => node.type === "tfoot");
const footerCells = footers[0].findAll((node) => node.type === "td");
// The Run time column in the footer is index 3 (Total / [empty Model] /
// Tokens / Run time / Billing).
const footerRuntime = textFromInstance(footerCells[3]);
expect(footerRuntime).toContain("30s");
} finally {
Date.now = originalNow;
}
});
});

161
apps/fabro-web/app/routes/run-billing.tsx
View file

@ -1,7 +1,11 @@
import { useMemo } from "react";
import { EmptyState } from "../components/state";
import { formatDurationSecs } from "../lib/format";
import { useRunBilling } from "../lib/queries";
import type { RunBilling } from "@qltysh/fabro-api-client";
import { IN_FLIGHT_STAGE_STATES } from "../lib/stage-sidebar";
import { useTickingNow } from "../lib/time";
import type { RunBilling, RunBillingStage } from "@qltysh/fabro-api-client";
const EMPTY_VALUE = "—";
@ -14,78 +18,103 @@ function formatUsdMicros(usdMicros?: number | null) {
return usdMicros == null ? EMPTY_VALUE : `$${(usdMicros / 1_000_000).toFixed(2)}`;
}
function mapBilling(billing: RunBilling | undefined) {
if (!billing) {
return {
stages: [],
totalRuntime: formatDurationSecs(0),
totalUsdMicros: undefined,
totalInput: null,
totalOutput: null,
modelBreakdown: [],
modelStageCount: 0,
};
}
function isInFlight(stage: RunBillingStage): boolean {
return stage.state != null && IN_FLIGHT_STAGE_STATES.has(stage.state);
}
const stages = billing.stages.map((stage) => {
const hasModel = stage.model != null;
return {
stage: stage.stage.name,
model: stage.model?.id ?? null,
inputTokens: hasModel ? stage.billing.input_tokens : null,
outputTokens: hasModel
? stage.billing.output_tokens + stage.billing.reasoning_tokens
: null,
runtime: formatDurationSecs(stage.runtime_secs),
totalUsdMicros: stage.billing.total_usd_micros,
};
});
const totalRuntime = formatDurationSecs(billing.totals.runtime_secs);
const hasLlmStages = billing.by_model.length > 0;
const totalInput = hasLlmStages ? billing.totals.input_tokens : null;
const totalOutput = hasLlmStages
? billing.totals.output_tokens + billing.totals.reasoning_tokens
: null;
const totalUsdMicros = billing.totals.total_usd_micros;
const modelBreakdown = billing.by_model
.map((entry) => ({
model: entry.model.id,
stages: entry.stages,
inputTokens: entry.billing.input_tokens,
outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens,
totalUsdMicros: entry.billing.total_usd_micros,
}))
.sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1));
const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0);
interface MappedStageRow {
stage: string;
model: string | null;
inputTokens: number | null;
outputTokens: number | null;
runtimeSecs: number;
totalUsdMicros: number | null | undefined;
}
function liveRuntimeSecs(stage: RunBillingStage, now: number): number {
if (stage.started_at) {
const startedMs = new Date(stage.started_at).getTime();
if (Number.isFinite(startedMs)) {
return Math.max(0, (now - startedMs) / 1000);
}
}
return stage.runtime_secs;
}
function mapStageRow(stage: RunBillingStage, runtimeSecs: number): MappedStageRow {
const hasModel = stage.model != null;
return {
stages,
totalRuntime,
totalUsdMicros,
totalInput,
totalOutput,
modelBreakdown,
modelStageCount,
stage: stage.stage.name,
model: stage.model?.id ?? null,
inputTokens: hasModel ? stage.billing.input_tokens : null,
outputTokens: hasModel
? stage.billing.output_tokens + stage.billing.reasoning_tokens
: null,
runtimeSecs,
totalUsdMicros: stage.billing.total_usd_micros,
};
}
export default function RunBilling({ params }: { params: { id: string } }) {
const billingQuery = useRunBilling(params.id);
const {
stages,
totalRuntime,
totalUsdMicros,
totalInput,
totalOutput,
modelBreakdown,
modelStageCount,
} = mapBilling(billingQuery.data);
const billing = billingQuery.data;
const hasInFlight = billing?.stages.some(isInFlight) ?? false;
if (!stages.length) {
// Tick once per second only while a stage is in-flight.
const now = useTickingNow(hasInFlight);
// Completed rows don't depend on `now`; memoize them by `billing` so we
// don't reallocate them every tick.
const completedRows = useMemo<MappedStageRow[]>(() => {
if (!billing) return [];
return billing.stages.map((stage) => mapStageRow(stage, stage.runtime_secs));
}, [billing]);
// The model breakdown is server-derived and stable across ticks too.
const modelBreakdown = useMemo(() => {
if (!billing) return [];
return billing.by_model
.map((entry) => ({
model: entry.model.id,
stages: entry.stages,
inputTokens: entry.billing.input_tokens,
outputTokens: entry.billing.output_tokens + entry.billing.reasoning_tokens,
totalUsdMicros: entry.billing.total_usd_micros,
}))
.sort((a, b) => (b.totalUsdMicros ?? -1) - (a.totalUsdMicros ?? -1));
}, [billing]);
// Re-derive only the in-flight rows on each tick; everything else stays put.
const rows = useMemo<MappedStageRow[]>(() => {
if (!billing) return [];
if (!hasInFlight) return completedRows;
return billing.stages.map((stage, idx) =>
isInFlight(stage)
? mapStageRow(stage, liveRuntimeSecs(stage, now))
: completedRows[idx],
);
}, [billing, completedRows, hasInFlight, now]);
// While ticking, sum the displayed row runtimes so the footer updates in
// lock-step. Otherwise trust the server's authoritative total.
const totalRuntimeSecs = hasInFlight
? rows.reduce((sum, row) => sum + row.runtimeSecs, 0)
: (billing?.totals.runtime_secs ?? 0);
const hasLlmStages = (billing?.by_model.length ?? 0) > 0;
const totalInput = hasLlmStages ? (billing?.totals.input_tokens ?? null) : null;
const totalOutput = hasLlmStages && billing
? billing.totals.output_tokens + billing.totals.reasoning_tokens
: null;
const totalUsdMicros = billing?.totals.total_usd_micros;
const modelStageCount = modelBreakdown.reduce((sum, row) => sum + row.stages, 0);
if (!rows.length) {
return (
<div className="py-12">
<EmptyState
title="No completed stages yet"
description="Stages will appear once the run produces completed nodes."
title="No stages yet"
description="Stages will appear as soon as the run starts executing."
/>
</div>
);
@ -105,7 +134,7 @@ export default function RunBilling({ params }: { params: { id: string } }) {
</tr>
</thead>
<tbody>
{stages.map((row) => (
{rows.map((row) => (
<tr key={row.stage} className="border-b border-line last:border-b-0">
<td className="px-4 py-3 text-fg-2">{row.stage}</td>
<td className="px-4 py-3 font-mono text-xs text-fg-3">
@ -115,7 +144,9 @@ export default function RunBilling({ params }: { params: { id: string } }) {
{formatTokens(row.inputTokens)} <span className="text-fg-muted">/</span>{" "}
{formatTokens(row.outputTokens)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs text-fg-3">{row.runtime}</td>
<td className="px-4 py-3 text-right font-mono text-xs text-fg-3">
{formatDurationSecs(row.runtimeSecs)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs text-fg-3">
{formatUsdMicros(row.totalUsdMicros)}
</td>
@ -131,7 +162,7 @@ export default function RunBilling({ params }: { params: { id: string } }) {
{formatTokens(totalOutput)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs font-medium text-fg">
{totalRuntime}
{formatDurationSecs(totalRuntimeSecs)}
</td>
<td className="px-4 py-3 text-right font-mono text-xs font-medium text-fg">
{formatUsdMicros(totalUsdMicros)}

34
apps/fabro-web/app/routes/run-overview.tsx
View file

@ -3,7 +3,6 @@ import { useNavigate, useParams } from "react-router";
import { graphTheme } from "../lib/graph-theme";
import { useRun, useRunGraph, useRunStages } from "../lib/queries";
import { StageSidebar } from "../components/stage-sidebar";
import type { Stage } from "../components/stage-sidebar";
import {
GRAPH_DEFAULT_ZOOM_INDEX,
GRAPH_ZOOM_STEPS,
@ -13,6 +12,7 @@ import { EmptyState } from "../components/state";
import {
ACTIVE_STAGE_STATES,
SUCCEEDED_STAGE_STATES,
aggregateGraphNodeStatus,
mapRunStagesToSidebarStages,
} from "../lib/stage-sidebar";
@ -54,27 +54,27 @@ export default function RunOverview() {
const inner = innerRef.current;
if (!inner || !graphSvg) return;
let cancelled = false;
(async () => {
if (cancelled) return;
inner.innerHTML = graphSvg;
const svg = inner.querySelector("svg");
if (!svg) return;
svgRef.current = svg;
const gt = graphTheme;
const runningDotIds = new Set<string>(
stages.filter((s: Stage) => ACTIVE_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id),
);
const failedDotIds = new Set<string>(
stages.filter((s: Stage) => s.status === "failed").map((s: Stage) => s.dotId ?? s.id),
);
const completedDotIds = new Set<string>(
stages.filter((s: Stage) => SUCCEEDED_STAGE_STATES.has(s.status)).map((s: Stage) => s.dotId ?? s.id),
);
const dotIdToStageId = new Map<string, string>(
stages.map((s: Stage) => [s.dotId ?? s.id, s.id]),
);
const aggregated = aggregateGraphNodeStatus(stages);
const runningDotIds = new Set<string>();
const failedDotIds = new Set<string>();
const completedDotIds = new Set<string>();
const dotIdToStageId = new Map<string, string>();
for (const [nodeId, { displayStatus, latestStageId }] of aggregated) {
dotIdToStageId.set(nodeId, latestStageId);
if (ACTIVE_STAGE_STATES.has(displayStatus)) {
runningDotIds.add(nodeId);
} else if (displayStatus === "failed") {
failedDotIds.add(nodeId);
} else if (SUCCEEDED_STAGE_STATES.has(displayStatus)) {
completedDotIds.add(nodeId);
}
}
const ns = "http://www.w3.org/2000/svg";
for (const group of svg.querySelectorAll(".node")) {
@ -148,8 +148,6 @@ export default function RunOverview() {
}
}
}
})();
return () => { cancelled = true; };
}, [stages, graphSvg, id, navigate, terminalOutcome]);
const onPointerDown = useCallback((e: React.PointerEvent) => {

191
apps/fabro-web/app/routes/run-stages.test.ts
View file

@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test";
import type { EventEnvelope } from "@qltysh/fabro-api-client";
import { isSafeMarkdownHref } from "./run-stages";
import { eventsToActivity, isSafeMarkdownHref } from "./run-stages";
describe("isSafeMarkdownHref", () => {
test("rejects protocol-relative URLs", () => {
@ -15,3 +16,191 @@ describe("isSafeMarkdownHref", () => {
expect(isSafeMarkdownHref("mailto:test@example.com")).toBe(true);
});
});
function envelope(seq: number, partial: Partial<EventEnvelope>): EventEnvelope {
return {
seq,
id: `evt-${seq}`,
ts: "2026-04-09T12:00:00Z",
run_id: "run-1",
event: "stage.prompt",
...partial,
} as EventEnvelope;
}
describe("eventsToActivity", () => {
test("filters events by stage_id (verify@1 vs verify@2 do not cross-contaminate)", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "stage.prompt",
stage_id: "verify@1",
node_id: "verify",
properties: { text: "first visit prompt" },
}),
envelope(2, {
event: "stage.prompt",
stage_id: "verify@2",
node_id: "verify",
properties: { text: "second visit prompt" },
}),
envelope(3, {
event: "agent.message",
stage_id: "verify@1",
node_id: "verify",
properties: { text: "first visit reply" },
}),
envelope(4, {
event: "agent.message",
stage_id: "verify@2",
node_id: "verify",
properties: { text: "second visit reply" },
}),
];
const firstVisit = eventsToActivity(events, "verify@1");
expect(firstVisit).toEqual([
{ kind: "system", content: "first visit prompt" },
{ kind: "assistant", content: "first visit reply" },
]);
const secondVisit = eventsToActivity(events, "verify@2");
expect(secondVisit).toEqual([
{ kind: "system", content: "second visit prompt" },
{ kind: "assistant", content: "second visit reply" },
]);
});
test("pairs command.started + command.completed into a single command turn", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "command.started",
node_id: "fmt",
properties: { script: "cargo fmt", language: "shell" },
}),
envelope(2, {
event: "command.completed",
node_id: "fmt",
properties: {
stdout: "ok",
stderr: "",
exit_code: 0,
duration_ms: 12,
termination: "exited",
},
}),
];
const turns = eventsToActivity(events, "fmt");
expect(turns).toHaveLength(1);
expect(turns[0]).toMatchObject({
kind: "command",
stageId: "fmt",
script: "cargo fmt",
language: "shell",
stdout: "ok",
exitCode: 0,
running: false,
});
});
test("command turn carries the requested stage_id, no @1 fallback", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "command.started",
stage_id: "verify@2",
node_id: "verify",
properties: { script: "echo hi", language: "shell" },
}),
envelope(2, {
event: "command.completed",
stage_id: "verify@2",
node_id: "verify",
properties: {
stdout: "hi",
stderr: "",
exit_code: 0,
duration_ms: 5,
termination: "exited",
},
}),
];
const turns = eventsToActivity(events, "verify@2");
expect(turns).toHaveLength(1);
const turn = turns[0];
expect(turn.kind).toBe("command");
if (turn.kind === "command") {
expect(turn.stageId).toBe("verify@2");
expect(turn.script).toBe("echo hi");
expect(turn.running).toBe(false);
}
});
test("pairs agent.tool.started + agent.tool.completed into a single tool turn", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "agent.tool.started",
node_id: "detect-drift",
properties: {
tool_call_id: "call-1",
tool_name: "read_file",
arguments: { path: "config.toml" },
},
}),
envelope(2, {
event: "agent.tool.completed",
node_id: "detect-drift",
properties: {
tool_call_id: "call-1",
tool_name: "read_file",
output: "[redis]",
is_error: false,
},
}),
];
const turns = eventsToActivity(events, "detect-drift");
expect(turns).toHaveLength(1);
expect(turns[0].kind).toBe("tool");
if (turns[0].kind === "tool") {
expect(turns[0].tools).toHaveLength(1);
expect(turns[0].tools[0]).toMatchObject({
id: "call-1",
toolName: "read_file",
result: "[redis]",
isError: false,
});
}
});
test("ignores unknown event types and events for other stages", () => {
const events: EventEnvelope[] = [
envelope(1, {
event: "stage.started",
node_id: "detect-drift",
properties: {},
}),
envelope(2, {
event: "agent.message",
node_id: "detect-drift",
properties: { text: "signal" },
}),
envelope(3, {
event: "run.running",
node_id: "detect-drift",
properties: {},
}),
envelope(4, {
event: "agent.message",
node_id: "other-stage",
properties: { text: "wrong stage" },
}),
];
const turns = eventsToActivity(events, "detect-drift");
expect(turns).toHaveLength(1);
if (turns[0].kind === "assistant") {
expect(turns[0].content).toBe("signal");
}
});
});

114
apps/fabro-web/app/routes/run-stages.tsx
View file

@ -40,16 +40,15 @@ import type { Stage } from "../components/stage-sidebar";
import { EmptyState } from "../components/state";
import { CopyButton } from "../components/ui";
import { formatDurationSecs } from "../lib/format";
import { fetchRunCommandLog, useRunEventsList, useRunStageTurns, useRunStages } from "../lib/queries";
import { mapRunStagesToSidebarStages } from "../lib/stage-sidebar";
import { useTickingNow } from "../lib/time";
import { fetchRunCommandLog, useRunStageEvents, useRunStages } from "../lib/queries";
import { STAGE_ACTIVITY_EVENT_TYPES, type StageActivityEventType } from "../lib/run-events";
import { ACTIVE_STAGE_STATES, formatStageLabel, mapRunStagesToSidebarStages } from "../lib/stage-sidebar";
import { getNumber, getString, type UnknownRecord } from "../lib/unknown";
import {
CommandOutputStream,
CommandTermination,
type EventEnvelope,
type StageTurn as ApiStageTurn,
type PaginatedStageTurnList,
type PaginatedEventList,
} from "@qltysh/fabro-api-client";
export const handle = { wide: true };
@ -68,17 +67,40 @@ function readTermination(props: UnknownRecord): CommandTermination {
return CommandTermination.EXITED;
}
function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
const stageEvents = events.filter((e) => e.node_id === stageId);
const STAGE_ACTIVITY_EVENT_SET = new Set<string>(STAGE_ACTIVITY_EVENT_TYPES);
function assertNever(value: never): never {
throw new Error(`Unhandled stage activity event type: ${value}`);
}
function activityEventStageId(event: EventEnvelope): string | undefined {
if (typeof event.stage_id === "string") return event.stage_id;
if (typeof event.node_id === "string") return event.node_id;
return getString(event.properties ?? {}, "node_id");
}
export function eventsToActivity(events: EventEnvelope[], stageId: string): TurnType[] {
const turns: TurnType[] = [];
// Collect tool pairs: started → completed
const pendingTools = new Map<string, { toolName: string; input: string }>();
// Track pending command for pairing started → completed
let pendingCommand: { stageId: string; script: string; language: string } | undefined;
for (const e of stageEvents) {
for (const e of events) {
const eventName = e.event;
if (
activityEventStageId(e) !== stageId ||
!eventName ||
!STAGE_ACTIVITY_EVENT_SET.has(eventName)
) {
continue;
}
// Exhaustive switch over StageActivityEventType: adding a new variant to
// STAGE_ACTIVITY_EVENT_TYPES forces a TS error here until the case is
// handled, keeping the SWR invalidation set and the reducer in sync.
const eventType = eventName as StageActivityEventType;
const props = e.properties ?? {};
switch (e.event) {
switch (eventType) {
case "stage.prompt":
turns.push({ kind: "system", content: getString(props, "text") ?? e.text ?? "" });
break;
@ -114,7 +136,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
}
case "command.started": {
pendingCommand = {
stageId: e.stage_id ?? `${stageId}@1`,
stageId,
script: getString(props, "script") ?? "",
language: getString(props, "language") ?? "shell",
};
@ -123,7 +145,7 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
case "command.completed": {
turns.push({
kind: "command",
stageId: pendingCommand?.stageId ?? e.stage_id ?? `${stageId}@1`,
stageId: pendingCommand?.stageId ?? stageId,
script: pendingCommand?.script ?? "",
language: pendingCommand?.language ?? "shell",
stdout: getString(props, "stdout") ?? "",
@ -136,6 +158,8 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
pendingCommand = undefined;
break;
}
default:
assertNever(eventType);
}
}
@ -153,41 +177,6 @@ function turnsFromEvents(events: EventEnvelope[], stageId: string): TurnType[] {
return turns;
}
function mapApiStageTurn(t: ApiStageTurn): TurnType {
switch (t.kind) {
case "tool":
return {
kind: "tool",
tools: (t.tools ?? []).map((tu) => ({
id: tu.id,
toolName: tu.tool_name,
input: tu.input,
result: tu.result,
isError: tu.is_error,
durationMs: tu.duration_ms,
})),
};
case "system":
case "assistant":
return { kind: t.kind, content: t.content ?? "" };
}
}
function mapTurns(
turnsResult: PaginatedStageTurnList | null | undefined,
eventsResult: PaginatedEventList | null | undefined,
selectedStageId: string | undefined,
): TurnType[] {
if (!selectedStageId) return [];
if (turnsResult?.data?.length) {
return turnsResult.data.map(mapApiStageTurn);
}
if (eventsResult?.data) {
return turnsFromEvents(eventsResult.data, selectedStageId);
}
return [];
}
function Markdown({ content }: { content: string }) {
const html = useMemo(() => markedSafe.parse(content, { async: false }) as string, [content]);
return (
@ -575,7 +564,6 @@ function RunningStageDuration({
const [startedAt, setStartedAt] = useState<number | null>(() =>
isRunning ? Date.now() : null,
);
const [, setTick] = useState(0);
useEffect(() => {
setStartedAt((current) => {
@ -584,14 +572,10 @@ function RunningStageDuration({
});
}, [isRunning]);
useEffect(() => {
if (!isRunning) return;
const interval = setInterval(() => setTick((tick) => tick + 1), 1000);
return () => clearInterval(interval);
}, [isRunning]);
const now = useTickingNow(isRunning);
if (isRunning && startedAt) {
return formatDurationSecs(Math.floor((Date.now() - startedAt) / 1000));
return formatDurationSecs(Math.floor((now - startedAt) / 1000));
}
return duration;
}
@ -605,16 +589,16 @@ export default function RunStages() {
);
const selectedStage = stages.find((s: Stage) => s.id === stageId) ?? stages[0];
const turnsQuery = useRunStageTurns(id, selectedStage?.id);
const hasStageTurns = (turnsQuery.data?.data.length ?? 0) > 0;
const shouldLoadEventFallback =
!!selectedStage?.id && !turnsQuery.isLoading && !turnsQuery.error && !hasStageTurns;
const eventsQuery = useRunEventsList(id, shouldLoadEventFallback);
const selectedStageId = selectedStage?.id;
const stageEventsQuery = useRunStageEvents(id, selectedStageId);
const turns = useMemo(
() => mapTurns(turnsQuery.data, eventsQuery.data, selectedStage?.id),
[eventsQuery.data, selectedStage?.id, turnsQuery.data],
() =>
selectedStageId
? eventsToActivity(stageEventsQuery.data ?? [], selectedStageId)
: [],
[stageEventsQuery.data, selectedStageId],
);
const isRunning = selectedStage?.status === "running";
const isActive = selectedStage ? ACTIVE_STAGE_STATES.has(selectedStage.status) : false;
if (!id || !stages.length) {
return (
@ -636,11 +620,13 @@ export default function RunStages() {
<div className="min-w-0 flex-1 space-y-3">
<div className="sticky top-0 z-10 -mx-2 flex items-center gap-2 bg-page/85 px-2 py-2 backdrop-blur">
<SelectedIcon className={`size-5 ${selectedConfig.color} ${isRunning ? "animate-spin" : ""}`} />
<h3 className="text-base font-semibold text-fg">{selectedStage.name}</h3>
<SelectedIcon className={`size-5 ${selectedConfig.color} ${isActive ? "animate-spin" : ""}`} />
<h3 className="text-base font-semibold text-fg">
{formatStageLabel(selectedStage)}
</h3>
<span className="font-mono text-xs tabular-nums text-fg-muted">
<RunningStageDuration
isRunning={isRunning}
isRunning={isActive}
duration={selectedStage.duration}
/>
</span>

1
apps/fabro-web/app/routes/runs.test.tsx
View file

@ -25,6 +25,7 @@ describe("runs route board mapping", () => {
test("keeps blocked runs in the blocked lane and preserves question text", () => {
const columns = buildBoardColumns({
columns: [
{ id: "queued", name: "Queued" },
{ id: "initializing", name: "Initializing" },
{ id: "running", name: "Running" },
{ id: "blocked", name: "Blocked" },

24
apps/fabro-web/app/routes/runs.tsx
View file

@ -18,7 +18,7 @@ import {
arrayMove,
} from "@dnd-kit/sortable";
import { CSS } from "@dnd-kit/utilities";
import { ciConfig, columnStatusDisplay, deriveCiStatus, mapRunListItem } from "../data/runs";
import { ciConfig, columnStatusDisplay, columnStatuses, deriveCiStatus, mapRunListItem } from "../data/runs";
import type { CiStatus, CheckRun, CheckStatus, RunItem, RunWithStatus, ColumnStatus } from "../data/runs";
import { EmptyState } from "../components/state";
import { SteerComposer } from "../components/steer-composer";
@ -38,6 +38,7 @@ interface ColumnStyle {
}
const columnStyles: Record<ColumnStatus, ColumnStyle> = {
queued: { iconType: "branch", actions: [] },
initializing: { iconType: "branch", actions: [] },
running: { iconType: "branch", actions: ["Watch", "Steer"] },
blocked: { iconType: "branch", actions: ["Answer Question"] },
@ -49,7 +50,7 @@ const defaultColumnStyle: ColumnStyle = { iconType: "branch", actions: [] };
const defaultColumnColors = { dot: "bg-fg-muted", text: "text-fg-muted" };
interface BoardRunsResponse {
columns: { id: string; name: string }[];
columns: PaginatedBoardRunList["columns"];
data: PaginatedBoardRunList["data"];
meta: PaginatedBoardRunList["meta"];
}
@ -64,16 +65,8 @@ type Column = {
items: RunItem[];
};
const SKELETON_STATUSES: ColumnStatus[] = [
"initializing",
"running",
"blocked",
"succeeded",
"failed",
];
function buildSkeletonColumns(): Column[] {
return SKELETON_STATUSES.map((id) => {
return columnStatuses.map((id) => {
const colors = columnStatusDisplay[id];
return {
id,
@ -98,7 +91,7 @@ export function buildBoardColumns(response: BoardRunsResponse): Column[] {
}
return response.columns.map((col) => {
const id = col.id as ColumnStatus;
const id = col.id;
const colors = columnStatusDisplay[id] ?? defaultColumnColors;
return {
id,
@ -768,6 +761,9 @@ export default function Runs() {
(sum, col) => sum + col.items.length,
0,
);
const visibleColumns = filteredColumns.filter(
(col) => col.id !== "queued" || col.items.length > 0,
);
return (
<DndContext sensors={sensors} collisionDetection={closestCenter} onDragEnd={handleDragEnd}>
@ -829,7 +825,7 @@ export default function Runs() {
{view === "columns" ? (
<>
<div className="flex gap-5 overflow-x-auto pb-4">
{filteredColumns.map((col) => (
{visibleColumns.map((col) => (
<div key={col.id} className="w-72 shrink-0">
<BoardColumn column={col} />
</div>
@ -852,7 +848,7 @@ export default function Runs() {
) : (
<>
<div className="space-y-4">
{filteredColumns.map((col) => {
{visibleColumns.map((col) => {
const isCollapsed = collapsed.has(col.id);
return (
<div key={col.id}>

189
docs/public/api-reference/fabro-api.yaml
View file

@ -1968,26 +1968,26 @@ paths:
schema:
$ref: "#/components/schemas/ErrorResponse"
/api/v1/runs/{id}/stages/{stageId}/turns:
/api/v1/runs/{id}/stages/{stageId}/events:
get:
operationId: listStageTurns
operationId: listStageEvents
tags: [Run Internals]
summary: List Stage Turns
description: Returns a paginated list of conversation turns within a specific stage, including system prompts, assistant responses, and tool invocations.
summary: List Stage Events
description: Returns a paginated JSON list of stored run events scoped to a single stage visit.
parameters:
- $ref: "#/components/parameters/RunId"
- $ref: "#/components/parameters/StageId"
- $ref: "#/components/parameters/PageLimit"
- $ref: "#/components/parameters/PageOffset"
- $ref: "#/components/parameters/SinceSeq"
- $ref: "#/components/parameters/EventLimit"
responses:
"200":
description: Paginated list of conversation turns
description: Paginated list of stage events
content:
application/json:
schema:
$ref: "#/components/schemas/PaginatedStageTurnList"
$ref: "#/components/schemas/PaginatedEventList"
"404":
description: Run or stage not found
description: Run not found.
headers:
x-request-id:
$ref: "#/components/headers/XRequestId"
@ -3918,20 +3918,6 @@ components:
meta:
$ref: "#/components/schemas/PaginationMeta"
PaginatedStageTurnList:
description: Paginated list of stage turns.
type: object
required:
- data
- meta
properties:
data:
type: array
items:
$ref: "#/components/schemas/StageTurn"
meta:
$ref: "#/components/schemas/PaginationMeta"
PaginatedApiQuestionList:
description: Paginated list of pending questions.
type: object
@ -5411,6 +5397,20 @@ components:
oneOf:
- $ref: "#/components/schemas/CommandTermination"
- type: "null"
started_at:
type: ["string", "null"]
format: date-time
description: Wall-clock time the latest attempt of this stage started, if known.
duration_ms:
type: ["integer", "null"]
format: uint64
minimum: 0
description: Wall-clock duration of the stage's latest terminal attempt, if known.
state:
oneOf:
- $ref: "#/components/schemas/StageState"
- type: "null"
description: Lifecycle state of the stage projection.
InterviewOption:
description: Option stored with an interview question in the event log.
@ -5739,6 +5739,7 @@ components:
description: Board column status for a run in the list view.
type: string
enum:
- queued
- initializing
- running
- blocked
@ -5752,7 +5753,7 @@ components:
- name
properties:
id:
type: string
$ref: "#/components/schemas/BoardColumn"
name:
type: string
@ -6242,7 +6243,7 @@ components:
example: 3501.0
BillingStageRef:
description: Reference to a billing stage.
description: Reference to a workflow node in a billing stage row.
type: object
required:
- id
@ -6402,11 +6403,13 @@ components:
- id
- name
- status
- node_id
- visit
properties:
id:
type: string
description: Unique stage identifier within the run.
example: propose-changes
description: StageId in "node_id@visit" form, e.g. verify@2.
example: verify@2
name:
type: string
description: Human-readable stage name.
@ -6417,107 +6420,21 @@ components:
type: number
description: Time spent in this stage, in seconds.
example: 154.0
dot_id:
node_id:
type: string
description: Node identifier in the Graphviz graph source.
example: propose
ToolUse:
description: A single tool invocation with its input, result, and execution metadata.
type: object
required:
- id
- tool_name
- input
- result
- is_error
properties:
id:
type: string
description: Unique identifier for this tool invocation. Enables correlation in parallel tool use.
example: toolu_01A09q90qw90lq917835lq9
tool_name:
type: string
description: Name of the tool that was invoked.
example: read_file
input:
type: string
description: JSON-encoded input passed to the tool.
example: '{ "path": "src/routes/auth.ts" }'
result:
type: string
description: Output returned by the tool. Contains the error message when is_error is true.
example: 'import { Router } from "express";'
is_error:
type: boolean
description: Whether the tool invocation failed. When true, the result field contains the error message.
example: false
duration_ms:
description: Node id in the workflow graph; multiple stages with different visits share the same node_id.
example: verify
visit:
type: integer
description: Wall-clock execution time of the tool invocation in milliseconds.
example: 142
StageTurn:
description: A single turn in a stage conversation — a system prompt, assistant response, or tool invocation block.
discriminator:
propertyName: kind
mapping:
system: "#/components/schemas/SystemStageTurn"
assistant: "#/components/schemas/AssistantStageTurn"
tool: "#/components/schemas/ToolStageTurn"
oneOf:
- $ref: "#/components/schemas/SystemStageTurn"
- $ref: "#/components/schemas/AssistantStageTurn"
- $ref: "#/components/schemas/ToolStageTurn"
SystemStageTurn:
description: A system prompt turn that sets the stage's instructions.
type: object
required:
- kind
- content
properties:
kind:
type: string
enum: [system]
content:
type: string
description: System prompt text.
example: You are a drift detection agent. Compare the production and staging environments.
AssistantStageTurn:
description: An assistant response turn within a stage.
type: object
required:
- kind
- content
properties:
kind:
type: string
enum: [assistant]
content:
type: string
description: Assistant response text.
example: I'll start by loading the environment configurations for both production and staging.
ToolStageTurn:
description: A tool invocation turn containing one or more tool calls.
type: object
required:
- kind
- tools
properties:
kind:
type: string
enum: [tool]
content:
type: string
description: Text accompanying the tool invocations, or null when the turn contains only tool calls.
tools:
type: array
description: Tool invocations executed in this turn.
items:
$ref: "#/components/schemas/ToolUse"
format: uint32
minimum: 1
description: 1-based visit count; bumped each time the workflow re-enters this node.
example: 2
started_at:
type: ["string", "null"]
format: date-time
description: Wall-clock time the latest attempt of this stage started, if known.
example: "2026-04-29T12:34:56Z"
# ── File Diff Schemas ──────────────────────────────────────────────
@ -6690,7 +6607,7 @@ components:
# ── Billing Schemas ──────────────────────────────────────────────────
RunBillingStage:
description: Token counts and billed totals for a single stage within a run.
description: Token counts and billed totals for one workflow node within a run. Rows are grouped by node; billing and runtime sum every visit of that node.
type: object
required:
- stage
@ -6701,7 +6618,7 @@ components:
stage:
$ref: "#/components/schemas/BillingStageRef"
model:
description: Model used for this stage; null for non-LLM stages.
description: Latest usage-bearing visit model for this node; null when no visit used an LLM model.
oneOf:
- $ref: "#/components/schemas/ModelReference"
- type: "null"
@ -6709,8 +6626,18 @@ components:
$ref: "#/components/schemas/BilledTokenCounts"
runtime_secs:
type: number
description: Wall-clock runtime in seconds.
description: Wall-clock runtime in seconds, summed across every visit of this node.
example: 154.0
started_at:
type: ["string", "null"]
format: date-time
description: Wall-clock time the latest attempt of this stage started, if known.
example: "2026-04-29T12:34:56Z"
state:
oneOf:
- $ref: "#/components/schemas/StageState"
- type: "null"
description: Lifecycle state of the stage. Use to detect in-flight rows for client-side runtime ticking.
RunBillingTotals:
description: Aggregate billing totals across all stages of a run.
@ -6770,7 +6697,7 @@ components:
$ref: "#/components/schemas/ModelReference"
stages:
type: integer
description: Number of stages that used this model.
description: Number of usage-bearing stage visits that used this model.
example: 2
billing:
$ref: "#/components/schemas/BilledTokenCounts"
@ -6785,7 +6712,7 @@ components:
properties:
stages:
type: array
description: Per-stage billing breakdown.
description: Per-node billing breakdown. Each row sums billing and runtime across all visits of that node.
items:
$ref: "#/components/schemas/RunBillingStage"
totals:

4
docs/public/reference/sdk.mdx
View file

@ -42,7 +42,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let config = SessionOptions::default();
let mut session = Session::new(client, profile, sandbox, config);
session.initialize().await;
session.initialize().await?;
// Subscribe to events before sending input
let mut events = session.subscribe();
@ -936,4 +936,4 @@ Register it on the client:
```rust
client.register_provider(Arc::new(MyProvider)).await?;
```
```

106
docs/superpowers/plans/2026-05-04-cross-tab-sse-coordination.md Normal file
View file

@ -0,0 +1,106 @@
# Cross-Tab SSE Coordination Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Coordinate Fabro web SSE subscriptions across tabs so one browser profile/origin opens at most one UI-owned `/api/v1/attach` EventSource in steady state when `BroadcastChannel` is available; brief overlap during election/takeover is tolerated and deduped.
**Architecture:** Add a browser-side SSE coordinator that elects one tab as leader, has that leader own the global EventSource, and broadcasts parsed run events to follower tabs over `BroadcastChannel`. Existing board and run-detail invalidation logic becomes a consumer of that global event feed, with the current per-tab SSE behavior preserved as a compatibility fallback.
**Tech Stack:** React, SWR, browser `BroadcastChannel`, browser `EventSource`, Bun tests, existing Fabro web API query keys.
---
## Summary
Build a browser-side SSE coordinator so Fabro web opens at most one `/api/v1/attach` EventSource per origin/browser profile in steady state when `BroadcastChannel` is available. Temporary duplicate leaders may exist during election/takeover, but event dedupe and generation checks make the overlap harmless and short-lived. The global stream becomes a shared cache-invalidation feed for both the runs board and run detail pages. No server API, OpenAPI, or Rust streaming contract changes are part of v1.
This supersedes the earlier web SSE limitation documented in `docs/plans/2026-04-19-002-feat-web-ui-lifecycle-actions-plan.md`: the old shared hook was code reuse only; this plan adds actual socket deduplication.
## Implementation Changes
- Add `apps/fabro-web/app/lib/cross-tab-sse.ts`.
- Export `subscribeToCrossTabSse(...)` with the same invalidation style as `subscribeToSharedEventSource`, a `resyncKeys` callback for gap recovery, and a `fallbackSubscribe` callback used when cross-tab coordination is unavailable.
- Use `BroadcastChannel` name `fabro:sse:v1`.
- Generate `tabId` with `crypto.randomUUID()` and a safe random fallback.
- Open one leader-owned `EventSource` to `queryKeys.system.attach()` (`/api/v1/attach`) in steady state.
- Leader dispatches each parsed `EventEnvelope` locally and broadcasts it to follower tabs.
- Followers do not open EventSource while a valid visible leader heartbeat exists; lower lexical `tabId` does not preempt a healthy visible leader.
- Implement leader election in the cross-tab module.
- Constants: `HEARTBEAT_MS = 1000`, `LEADER_STALE_MS = 4000`, `ELECTION_JITTER_MS = 150`.
- Messages: `hello`, `heartbeat`, `candidate`, `leader-changed`, `release`, `resync`, `event`.
- Define a typed message union. Every message includes `type`, `version: 1`, `tabId`, and `sentAt`.
- `heartbeat`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`.
- `candidate`: `{ type, version, tabId, sentAt, candidateId: tabId, candidateGeneration, visibility, observedLeaderId, observedGeneration, reason }`, where `reason` is `"hidden-leader" | "stale-leader" | "release" | "no-leader"`.
- `leader-changed`: `{ type, version, tabId, sentAt, leaderId, generation, visibility }`.
- `release`: `{ type, version, tabId, sentAt, leaderId, generation }`.
- `resync`: `{ type, version, tabId, sentAt, leaderId, generation, reason }`.
- `event`: `{ type, version, tabId, sentAt, leaderId, generation, payload }`.
- Use the candidacy phase for all leadership changes: hidden-leader takeover, stale-leader recovery, leader release, and no-leader startup.
- A candidate sets `candidateGeneration = observedGeneration + 1`, broadcasts `candidate`, waits jitter, and opens EventSource only if no higher-priority candidate for the same `candidateGeneration` appears.
- Candidate priority is election-scoped: visible candidates outrank hidden candidates; for equal visibility, lower lexical `candidateId` wins. This priority resolves elections and same-generation split brain only; it is not a reason to preempt a fresh visible leader.
- When a visible follower observes a fresh hidden leader heartbeat, it enters candidacy with `reason: "hidden-leader"`.
- When tabs detect a stale leader, leader release, or no known leader, they enter the same candidacy flow with the matching `reason`.
- If two visible candidates race for the same observed leader/generation, the lower lexical `candidateId` wins.
- Current leaders release when they observe a candidate whose `observedLeaderId` matches their `leaderId` and whose `observedGeneration` is current or newer.
- If same-generation split brain still occurs, lower-priority leaders release when they observe a same-generation higher-priority leader heartbeat or `leader-changed`.
- Each new leader uses `candidateGeneration`, broadcasts `leader-changed`, and followers ignore heartbeats/events from non-current leaders or stale generations.
- Hidden leader keeps the stream only when no visible candidate takes over.
- On `pagehide`/last local unsubscribe, a leader closes EventSource and broadcasts `release`.
- Brief split brain is tolerated; dedupe events by `payload.id`, falling back to `${run_id}:${seq}:${event}`.
- Keep dedupe bounded with a recent-event cache: max 1000 IDs and 5 minute TTL. Evict oldest entries when the max is exceeded and prune expired entries during event handling. Duplicate invalidations after eviction are acceptable; unbounded growth is not.
- Migrate consumers.
- `apps/fabro-web/app/lib/board-events.ts`: subscribe through the cross-tab global stream; keep existing board event allowlist.
- `apps/fabro-web/app/lib/run-events.ts`: subscribe through the same global stream, filter by `payload.run_id === runId`, and reuse `queryKeysForRunEvent`.
- In coordinated mode, run detail pages stay subscribed while mounted, including terminal runs, so post-terminal archive/unarchive changes can reconcile live.
- Do not close the global stream on `run.completed` / `run.failed`; terminal events only invalidate run-scoped keys.
- Keep `subscribeToSharedEventSource` in `apps/fabro-web/app/lib/sse.ts` for fallback and existing local sharing behavior.
- Gap and fallback behavior.
- If `BroadcastChannel` is unavailable or throws, call each subscriber's `fallbackSubscribe`.
- Board fallback uses the existing global `/api/v1/attach` path.
- Run detail fallback preserves the existing run-scoped `/api/v1/runs/:id/attach` path, so the old terminal-tab stale limitation remains only in fallback mode.
- Do not add replay to `/api/v1/attach`.
- On leader takeover, stale leader timeout, leader release, and new leader generation, broadcast `resync` or `leader-changed` so every tab with active local subscriptions runs its own `resyncKeys`.
- On `visibilitychange` back to visible without leadership change, run only that tab's local `resyncKeys`; do not broadcast cross-tab resync.
- Board `resyncKeys`: `queryKeys.boards.runs()`.
- Run `resyncKeys`: detail, files, billing, stages, events, LR graph, TB graph, and questions for that run.
## Tests
- Add `apps/fabro-web/app/lib/cross-tab-sse.test.ts` with fake `BroadcastChannel`, fake `EventSource`, and fake timers.
- One leader opens `/api/v1/attach`; followers open no EventSource.
- Leader broadcasts an event and all local subscribers receive invalidations.
- Run subscribers ignore events for other `run_id` values.
- Board and run subscriptions can coexist on the same global stream.
- Temporary duplicate leaders are allowed only during election/takeover and converge back to one leader.
- With fresh hidden-leader heartbeats, a visible tab broadcasts candidacy, hidden leader closes, visible tab opens `/api/v1/attach`, and followers resync.
- Two visible candidates racing for the same hidden leader resolve to the lower lexical `candidateId`.
- Two tabs detect the same stale leader simultaneously; only the winning candidate opens `/api/v1/attach` after jitter.
- Same-generation split brain converges to one leader by visibility, then lexical `tabId`.
- A fresh visible leader exists; a new visible follower with a lower lexical `tabId` joins and does not take leadership.
- A hidden leader does not release for a candidate that references an old `observedLeaderId` or stale `observedGeneration`.
- Stale heartbeat triggers takeover and every active tab calls its own `resyncKeys`.
- A follower tab calls its own board/run `resyncKeys` after another tab becomes leader.
- Duplicate event IDs are ignored.
- The recent-event dedupe cache evicts by TTL and max-size bound; duplicate invalidation may recur only after eviction.
- Stale heartbeat/event messages from an old leader/generation are ignored after takeover.
- Last unsubscribe closes leader EventSource and sends `release`.
- Missing/broken BroadcastChannel uses the per-tab fallback.
- Update existing tests.
- `apps/fabro-web/app/lib/board-events.test.tsx`: assert coordinated mode uses `/api/v1/attach` once and fallback preserves current behavior.
- `apps/fabro-web/app/lib/run-events.test.tsx`: assert coordinated mode filters by `run_id`; fallback preserves current `/runs/:id/attach` behavior; terminal run events do not close the global coordinator; terminal detail tabs still receive archive/unarchive invalidations while mounted in coordinated mode.
- Verification commands.
- `cd apps/fabro-web && bun test app/lib/cross-tab-sse.test.ts app/lib/board-events.test.tsx app/lib/run-events.test.tsx`
- `cd apps/fabro-web && bun run typecheck`
- Optional manual check: open 8 run-detail tabs for active runs; after election settles and with BroadcastChannel available, Chrome should show one active `/api/v1/attach` EventStream across the participating tabs and no UI-created `/api/v1/runs/:id/attach` streams.
## Assumptions
- Treat SSE as live cache invalidation, not an exact gapless event log.
- Preserve current run-specific attach endpoint for CLI, API clients, and fallback only.
- Do not touch unrelated dirty worktree files; intentionally replace or adapt the earlier interrupted SSE test edits as part of the new test suite.
- No docs or public API updates are required for v1 because this is an internal web transport change.

4
lib/crates/fabro-agent/README.md
View file

@ -140,7 +140,7 @@ let config = SessionConfig {
// 5. Create and initialize the session
let mut session = Session::new(client, profile, env, config, None);
session.initialize().await;
session.initialize().await?;
// 6. Subscribe to events (for UI rendering)
let mut rx = session.subscribe();
@ -232,4 +232,4 @@ profile.register_subagent_tools(manager, factory, 0);
- **Tool output truncation** -- Per-tool character and line limits with head/tail or tail-only truncation modes
- **Environment variable filtering** -- `LocalSandbox` strips secrets (`*_API_KEY`, `*_SECRET`, `*_TOKEN`, `*_PASSWORD`, `*_CREDENTIAL`) from subprocess environments
- **Command timeouts** -- Configurable per-command with process group cleanup (SIGTERM then SIGKILL)
- **Project doc discovery** -- Automatically discovers `AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, or `.codex/instructions.md` based on provider, with a 32KB budget
- **Project doc discovery** -- Automatically discovers `AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, or `.codex/instructions.md` based on provider, with a 32KB budget

2
lib/crates/fabro-agent/src/cli.rs
View file

@ -724,7 +724,7 @@ pub async fn run_with_args_and_client(
});
// Initialize and run
session.initialize().await;
session.initialize().await?;
let result = session.process_input(&args.prompt).await;
if matches!(output_format, OutputFormat::Text) {

100
lib/crates/fabro-agent/src/memory.rs
View file

@ -1,8 +1,10 @@
use std::collections::HashSet;
use fabro_model::Provider;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use crate::error::{Error, InterruptReason};
use crate::sandbox::Sandbox;
const BUDGET_BYTES: usize = 32768;
@ -12,7 +14,8 @@ pub async fn discover_memory(
git_root: &str,
working_dir: &str,
provider: Provider,
) -> Vec<String> {
cancel_token: &CancellationToken,
) -> Result<Vec<String>, Error> {
let directories = build_directory_walk(git_root, working_dir);
let candidate_filenames: Vec<&str> = match provider {
@ -34,8 +37,15 @@ pub async fn discover_memory(
for dir in &directories {
for filename in &candidate_filenames {
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let path = format!("{dir}/{filename}");
if let Ok(content) = env.read_file(&path, None, None).await {
let read_result = env.read_file(&path, None, None).await;
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
if let Ok(content) = read_result {
if content.is_empty() {
warn!(path = %path, "Project doc file empty, skipping");
continue;
@ -68,7 +78,7 @@ pub async fn discover_memory(
let total_bytes: usize = results.iter().map(std::string::String::len).sum();
info!(files = results.len(), total_bytes, "Project docs loaded");
results
Ok(results)
}
fn build_directory_walk(git_root: &str, working_dir: &str) -> Vec<String> {
@ -117,6 +127,8 @@ mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use tokio_util::sync::CancellationToken;
use super::*;
use crate::sandbox::Sandbox;
use crate::test_support::MockSandbox;
@ -129,7 +141,15 @@ mod tests {
files,
..Default::default()
});
let docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await;
let docs = discover_memory(
env.as_ref(),
"/repo",
"/repo",
Provider::Anthropic,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(docs.len(), 1);
assert_eq!(docs[0], "Agent instructions");
}
@ -146,8 +166,15 @@ mod tests {
files: files.clone(),
..Default::default()
});
let anthropic_docs =
discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await;
let anthropic_docs = discover_memory(
env.as_ref(),
"/repo",
"/repo",
Provider::Anthropic,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(anthropic_docs.len(), 2);
assert_eq!(anthropic_docs[0], "agents");
assert_eq!(anthropic_docs[1], "claude");
@ -156,7 +183,15 @@ mod tests {
files: files.clone(),
..Default::default()
});
let openai_docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::OpenAi).await;
let openai_docs = discover_memory(
env.as_ref(),
"/repo",
"/repo",
Provider::OpenAi,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(openai_docs.len(), 2);
assert_eq!(openai_docs[0], "agents");
assert_eq!(openai_docs[1], "copilot");
@ -165,7 +200,15 @@ mod tests {
files,
..Default::default()
});
let gemini_docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Gemini).await;
let gemini_docs = discover_memory(
env.as_ref(),
"/repo",
"/repo",
Provider::Gemini,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(gemini_docs.len(), 2);
assert_eq!(gemini_docs[0], "agents");
assert_eq!(gemini_docs[1], "gemini");
@ -184,7 +227,15 @@ mod tests {
files,
..Default::default()
});
let docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await;
let docs = discover_memory(
env.as_ref(),
"/repo",
"/repo",
Provider::Anthropic,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(docs.len(), 2);
assert_eq!(docs[0], large_content);
// Second doc should be truncated to fit remaining budget
@ -201,7 +252,15 @@ mod tests {
files,
..Default::default()
});
let docs = discover_memory(env.as_ref(), "/repo", "/repo", Provider::Anthropic).await;
let docs = discover_memory(
env.as_ref(),
"/repo",
"/repo",
Provider::Anthropic,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(docs.len(), 1);
assert_eq!(docs[0], "shared instructions");
}
@ -215,7 +274,15 @@ mod tests {
files,
..Default::default()
});
let docs = discover_memory(env.as_ref(), "/repo", "/repo/src", Provider::Anthropic).await;
let docs = discover_memory(
env.as_ref(),
"/repo",
"/repo/src",
Provider::Anthropic,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(docs.len(), 1);
assert_eq!(docs[0], "shared instructions");
}
@ -231,8 +298,15 @@ mod tests {
files,
..Default::default()
});
let docs =
discover_memory(env.as_ref(), "/repo", "/repo/src/app", Provider::Anthropic).await;
let docs = discover_memory(
env.as_ref(),
"/repo",
"/repo/src/app",
Provider::Anthropic,
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(docs.len(), 3);
assert_eq!(docs[0], "root agents");
assert_eq!(docs[1], "src agents");

212
lib/crates/fabro-agent/src/session.rs
View file

@ -250,13 +250,24 @@ impl Session {
/// Initialize session by discovering project docs and capturing environment
/// context. Call before `process_input`.
pub async fn initialize(&mut self) {
///
/// # Errors
///
/// Returns `Error::Interrupted(InterruptReason::Cancelled)` if the
/// session's cancel token fires during initialization.
pub async fn initialize(&mut self) -> Result<(), Error> {
let cancel_token = self.cancel_token.clone();
self.event_emitter
.emit(self.id.clone(), AgentEvent::SessionStarted {
provider: Some(self.provider_profile.provider().to_string()),
model: Some(self.provider_profile.model().to_string()),
});
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let doc_root = self
.config
.git_root
@ -267,8 +278,9 @@ impl Session {
&doc_root,
self.sandbox.working_directory(),
self.provider_profile.provider(),
&cancel_token,
)
.await;
.await?;
// Discover skills
let skill_dirs = if let Some(dirs) = &self.config.skill_dirs {
@ -278,7 +290,7 @@ impl Session {
let skills_str = skills_dir.to_string_lossy().to_string();
default_skill_dirs(Some(&skills_str), self.config.git_root.as_deref())
};
self.skills = discover_skills(self.sandbox.as_ref(), &skill_dirs).await;
self.skills = discover_skills(self.sandbox.as_ref(), &skill_dirs, &cancel_token).await?;
debug!(skill_count = self.skills.len(), "Skills discovered");
// Register use_skill tool when skills are available
@ -295,7 +307,7 @@ impl Session {
if !self.config.mcp_servers.is_empty() {
// Resolve Sandbox transports: start the server inside the sandbox,
// then rewrite the config to Http using the sandbox's preview URL.
let mcp_servers = self.resolve_sandbox_mcp_servers().await;
let mcp_servers = self.resolve_sandbox_mcp_servers(&cancel_token).await?;
let mut manager = McpConnectionManager::new();
let results = manager.start_servers(&mcp_servers).await;
@ -329,7 +341,7 @@ impl Session {
}
// Populate environment context
self.env_context = self.build_env_context().await;
self.env_context = self.build_env_context(&cancel_token).await?;
debug!(
is_git_repo = self.env_context.is_git_repo,
model = %self.env_context.model,
@ -344,19 +356,30 @@ impl Session {
self.config.user_instructions.as_deref(),
&self.skills,
);
Ok(())
}
/// Resolve `McpTransport::Sandbox` configs by starting the MCP server
/// inside the sandbox and rewriting the transport to `Http` with the
/// sandbox's preview URL.
async fn resolve_sandbox_mcp_servers(&self) -> Vec<McpServerSettings> {
async fn resolve_sandbox_mcp_servers(
&self,
cancel_token: &CancellationToken,
) -> Result<Vec<McpServerSettings>, Error> {
let mut resolved = Vec::with_capacity(self.config.mcp_servers.len());
for config in &self.config.mcp_servers {
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
match &config.transport {
McpTransport::Sandbox { command, port, env } => {
let port = *port;
match self.start_sandbox_mcp_server(command, port, env).await {
match self
.start_sandbox_mcp_server(command, port, env, cancel_token)
.await?
{
Ok((url, headers)) => {
info!(
server = %config.name,
@ -388,17 +411,24 @@ impl Session {
}
}
resolved
Ok(resolved)
}
/// Start an MCP server inside the sandbox and return (url, headers) for
/// HTTP connection.
///
/// The outer `Result` surfaces fatal cancellation as
/// `Error::Interrupted(InterruptReason::Cancelled)` (the running MCP
/// process group is terminated before returning). The inner `Result`
/// captures non-fatal startup failures that the caller logs and turns
/// into an `McpServerFailed` event.
async fn start_sandbox_mcp_server(
&self,
command: &[String],
port: u16,
env: &std::collections::HashMap<String, String>,
) -> Result<(String, std::collections::HashMap<String, String>), String> {
cancel_token: &CancellationToken,
) -> Result<Result<(String, std::collections::HashMap<String, String>), String>, Error> {
let sandbox = self.sandbox.as_ref();
let cmd_str = command
@ -416,27 +446,63 @@ impl Session {
quoted = fabro_sandbox::shell_quote(&inner)
);
let env_ref = if env.is_empty() { None } else { Some(env) };
let launch_result = sandbox
.exec_command(&launch_script, 30_000, None, env_ref, None)
.await
.map_err(|e| format!("Failed to launch MCP server: {}", e.display_with_causes()))?;
let pid = launch_result.stdout.trim();
info!(pid, port, "MCP server process launched in sandbox");
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let launch_result = match sandbox
.exec_command(
&launch_script,
30_000,
None,
env_ref,
Some(cancel_token.child_token()),
)
.await
{
Ok(result) => result,
Err(e) => {
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
return Ok(Err(format!(
"Failed to launch MCP server: {}",
e.display_with_causes()
)));
}
};
let pid = launch_result.stdout.trim().to_string();
info!(pid = %pid, port, "MCP server process launched in sandbox");
// Wait for the server to start listening on the port
let poll_cmd = format!(
"for i in $(seq 1 30); do ss -tln | grep -q ':{port} ' && echo ready && exit 0; sleep 1; done; echo timeout"
);
let poll_result = sandbox
.exec_command(&poll_cmd, 60_000, None, None, None)
.await
.map_err(|e| {
format!(
.exec_command(
&poll_cmd,
60_000,
None,
None,
Some(cancel_token.child_token()),
)
.await;
if cancel_token.is_cancelled() {
kill_mcp_pid(sandbox, &pid).await;
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let poll_result = match poll_result {
Ok(result) => result,
Err(e) => {
return Ok(Err(format!(
"Failed to poll MCP server readiness: {}",
e.display_with_causes()
)
})?;
)));
}
};
if poll_result.stdout.trim() != "ready" {
// Grab stderr for debugging
@ -446,51 +512,80 @@ impl Session {
10_000,
None,
None,
None,
Some(cancel_token.child_token()),
)
.await
.map(|r| r.stdout)
.unwrap_or_default();
return Err(format!(
return Ok(Err(format!(
"MCP server did not start listening on port {port} within 30s. stderr:\n{stderr}"
));
)));
}
// Get the preview URL for the port, or fall back to localhost for local
// sandboxes
if let Some(url_and_headers) = sandbox
.get_preview_url(port)
.await
.map_err(|e| e.display_with_causes())?
{
Ok(url_and_headers)
let preview = match sandbox.get_preview_url(port).await {
Ok(p) => p,
Err(e) => return Ok(Err(e.display_with_causes())),
};
if cancel_token.is_cancelled() {
kill_mcp_pid(sandbox, &pid).await;
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
if let Some(url_and_headers) = preview {
Ok(Ok(url_and_headers))
} else {
info!(port, "No preview URL available, using localhost");
Ok((
Ok(Ok((
format!("http://localhost:{port}"),
std::collections::HashMap::new(),
))
)))
}
}
async fn build_env_context(&self) -> EnvContext {
async fn build_env_context(
&self,
cancel_token: &CancellationToken,
) -> Result<EnvContext, Error> {
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
let model_name = self.provider_profile.model().to_string();
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
// Detect git info via sandbox
let git_branch = self
.sandbox
.exec_command("git rev-parse --abbrev-ref HEAD", 5000, None, None, None)
.exec_command(
"git rev-parse --abbrev-ref HEAD",
5000,
None,
None,
Some(cancel_token.child_token()),
)
.await
.ok()
.filter(fabro_sandbox::ExecResult::is_success)
.map(|r| r.stdout.trim().to_string());
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let is_git_repo = git_branch.is_some();
let git_status_short = if is_git_repo {
self.sandbox
.exec_command("git status --short", 5000, None, None, None)
.exec_command(
"git status --short",
5000,
None,
None,
Some(cancel_token.child_token()),
)
.await
.ok()
.filter(fabro_sandbox::ExecResult::is_success)
@ -500,9 +595,19 @@ impl Session {
None
};
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let git_recent_commits = if is_git_repo {
self.sandbox
.exec_command("git log --oneline -10", 5000, None, None, None)
.exec_command(
"git log --oneline -10",
5000,
None,
None,
Some(cancel_token.child_token()),
)
.await
.ok()
.filter(fabro_sandbox::ExecResult::is_success)
@ -512,7 +617,11 @@ impl Session {
None
};
EnvContext {
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
Ok(EnvContext {
git_branch,
is_git_repo,
current_date: today,
@ -520,7 +629,7 @@ impl Session {
knowledge_cutoff: self.provider_profile.knowledge_cutoff().unwrap_or_default(),
git_status_short,
git_recent_commits,
}
})
}
#[must_use]
@ -1290,6 +1399,23 @@ const fn is_auth_error(err: &LlmError) -> bool {
)
}
/// Best-effort kill of a sandbox MCP server process group. Used when
/// `start_sandbox_mcp_server` is cancelled after spawning a detached
/// `setsid` child but before reporting readiness. Errors from the sandbox
/// are logged and swallowed; the caller is already returning a Cancelled
/// error.
async fn kill_mcp_pid(sandbox: &dyn Sandbox, pid: &str) {
let pid = pid.trim();
if pid.is_empty() {
return;
}
let script =
format!("kill -TERM -{pid} 2>/dev/null; sleep 1; kill -KILL -{pid} 2>/dev/null; true");
if let Err(err) = sandbox.exec_command(&script, 5_000, None, None, None).await {
warn!(pid, error = %err.display_with_causes(), "Failed to kill MCP server process group during cancellation");
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
@ -1648,7 +1774,7 @@ mod tests {
let mut session = make_session(vec![text_response("Hello")]).await;
let mut rx = session.subscribe();
session.initialize().await;
session.initialize().await.unwrap();
session.process_input("Hi").await.unwrap();
session.close();
@ -2176,7 +2302,7 @@ mod tests {
let mut session = make_session(responses).await;
let mut rx = session.subscribe();
session.initialize().await;
session.initialize().await.unwrap();
session.process_input("one").await.unwrap();
session.process_input("two").await.unwrap();
session.close();
@ -2209,7 +2335,7 @@ mod tests {
..Default::default()
};
let mut session = Session::new(client, profile, env, config, None);
session.initialize().await;
session.initialize().await.unwrap();
session.process_input("test").await.unwrap();
// Verify user instructions are included in the system prompt
@ -2998,7 +3124,7 @@ mod tests {
let mut rx = session.subscribe();
// Initialize starts the MCP server and registers tools
session.initialize().await;
session.initialize().await.unwrap();
// Verify McpServerReady event was emitted
let mut mcp_ready = false;
@ -3193,7 +3319,7 @@ mod tests {
#[tokio::test]
async fn process_input_emits_processing_end_on_idle_transition() {
let mut session = make_session(vec![text_response("Hello")]).await;
session.initialize().await;
session.initialize().await.unwrap();
let mut rx = session.subscribe();
session.process_input("Hi").await.unwrap();

48
lib/crates/fabro-agent/src/skills.rs
View file

@ -1,7 +1,9 @@
use std::sync::Arc;
use fabro_llm::types::ToolDefinition;
use tokio_util::sync::CancellationToken;
use crate::error::{Error, InterruptReason};
use crate::sandbox::Sandbox;
use crate::tool_registry::RegisteredTool;
use crate::tools::required_str;
@ -224,17 +226,35 @@ pub fn default_skill_dirs(fabro_skills_dir: Option<&str>, git_root: Option<&str>
dirs
}
pub async fn discover_skills(env: &dyn Sandbox, dirs: &[String]) -> Vec<Skill> {
pub async fn discover_skills(
env: &dyn Sandbox,
dirs: &[String],
cancel_token: &CancellationToken,
) -> Result<Vec<Skill>, Error> {
let mut skills_by_name: std::collections::HashMap<String, Skill> =
std::collections::HashMap::new();
for dir in dirs {
let Ok(paths) = env.glob("*/SKILL.md", Some(dir)).await else {
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let glob_result = env.glob("*/SKILL.md", Some(dir)).await;
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let Ok(paths) = glob_result else {
continue;
};
for path in paths {
let Ok(content) = env.read_file(&path, None, None).await else {
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let read_result = env.read_file(&path, None, None).await;
if cancel_token.is_cancelled() {
return Err(Error::Interrupted(InterruptReason::Cancelled));
}
let Ok(content) = read_result else {
continue;
};
@ -246,7 +266,7 @@ pub async fn discover_skills(env: &dyn Sandbox, dirs: &[String]) -> Vec<Skill> {
let mut skills: Vec<Skill> = skills_by_name.into_values().collect();
skills.sort_by(|a, b| a.name.cmp(&b.name));
skills
Ok(skills)
}
#[cfg(test)]
@ -454,7 +474,9 @@ name: trimmed
..Default::default()
};
let skills = discover_skills(&env, &["/skills".into()]).await;
let skills = discover_skills(&env, &["/skills".into()], &CancellationToken::new())
.await
.unwrap();
assert_eq!(skills.len(), 1);
assert_eq!(skills[0].name, "commit");
assert_eq!(skills[0].description, "Make a commit");
@ -477,7 +499,9 @@ name: trimmed
..Default::default()
};
let skills = discover_skills(&env, &["/skills".into()]).await;
let skills = discover_skills(&env, &["/skills".into()], &CancellationToken::new())
.await
.unwrap();
assert_eq!(skills.len(), 1);
assert_eq!(skills[0].name, "good");
}
@ -485,7 +509,9 @@ name: trimmed
#[tokio::test]
async fn discover_empty_dirs() {
let env = MockSandbox::default();
let skills = discover_skills(&env, &[]).await;
let skills = discover_skills(&env, &[], &CancellationToken::new())
.await
.unwrap();
assert!(skills.is_empty());
}
@ -514,7 +540,13 @@ name: trimmed
};
// discover_skills iterates dirs in order; later dirs override earlier names
let skills = discover_skills(&env, &["/global".into(), "/project".into()]).await;
let skills = discover_skills(
&env,
&["/global".into(), "/project".into()],
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(skills.len(), 1);
assert_eq!(skills[0].description, "Project commit");
}

2
lib/crates/fabro-agent/src/subagent.rs
View file

@ -111,7 +111,7 @@ impl SubAgentManager {
let task_prompt_for_spawn = task_prompt.clone();
let task = tokio::spawn(async move {
session.initialize().await;
session.initialize().await?;
session.process_input(&task_prompt_for_spawn).await?;
let turns = session.history().turns();
let last_text = turns.iter().rev().find_map(|t| match t {

2
lib/crates/fabro-agent/src/v4a_patch.rs
View file

@ -1466,7 +1466,7 @@ def farewell(name):
SessionOptions::default(),
None,
);
session.initialize().await;
session.initialize().await.unwrap();
session
.process_input("Update the greeting functions")
.await

8
lib/crates/fabro-agent/tests/it/parity_matrix.rs
View file

@ -170,7 +170,7 @@ macro_rules! provider_test {
async fn [<$prefix _ $scenario>]() {
let tmp = tempfile::tempdir().expect("failed to create tempdir");
let mut session = make_session($provider, $model, tmp.path(), None).await;
session.initialize().await;
session.initialize().await.unwrap();
[<scenario_ $scenario>](&mut session, tmp.path()).await;
}
}
@ -195,7 +195,7 @@ macro_rules! openai_twin_provider_test {
tmp.path(),
Some(twin),
).await;
session.initialize().await;
session.initialize().await.unwrap();
[<scenario_ $scenario>](&mut session, tmp.path()).await;
}
}
@ -668,7 +668,7 @@ macro_rules! reasoning_effort_tests {
};
let mut session =
make_session_with_config($provider, $model, tmp.path(), config, None).await;
session.initialize().await;
session.initialize().await.unwrap();
session
.process_input("Say hello")
.await
@ -747,7 +747,7 @@ macro_rules! loop_detection_tests {
};
let mut session =
make_session_with_config($provider, $model, tmp.path(), config, None).await;
session.initialize().await;
session.initialize().await.unwrap();
session
.process_input("Repeatedly read the file /dev/null")
.await

57
lib/crates/fabro-api/tests/run_billing_stage_round_trip.rs
View file

@ -1,4 +1,5 @@
use fabro_api::types::RunBillingStage;
use fabro_types::StageState;
use serde_json::json;
#[test]
@ -28,3 +29,59 @@ fn run_billing_stage_model_accepts_required_null() {
assert!(encoded.get("model").is_some());
assert!(encoded["model"].is_null());
}
#[test]
fn run_billing_stage_round_trips_terminal_row_with_started_at_and_state() {
let value = json!({
"stage": {
"id": "build",
"name": "build"
},
"model": { "id": "claude-sonnet-4-5" },
"billing": {
"input_tokens": 12,
"output_tokens": 34,
"total_tokens": 46,
"reasoning_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0
},
"runtime_secs": 5.5,
"started_at": "2026-04-29T12:34:56Z",
"state": "succeeded"
});
let stage: RunBillingStage =
serde_json::from_value(value.clone()).expect("terminal stage row should deserialize");
assert!(stage.started_at.is_some());
assert_eq!(stage.state, Some(StageState::Succeeded));
assert_eq!(serde_json::to_value(stage).unwrap(), value);
}
#[test]
fn run_billing_stage_round_trips_in_flight_row() {
let value = json!({
"stage": {
"id": "build",
"name": "build"
},
"model": null,
"billing": {
"input_tokens": 0,
"output_tokens": 0,
"total_tokens": 0,
"reasoning_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0
},
"runtime_secs": 1.25,
"started_at": "2026-04-29T12:34:56Z",
"state": "running"
});
let stage: RunBillingStage =
serde_json::from_value(value.clone()).expect("in-flight stage row should deserialize");
assert!(stage.model.is_none());
assert_eq!(stage.state, Some(StageState::Running));
assert_eq!(serde_json::to_value(stage).unwrap(), value);
}

5
lib/crates/fabro-api/tests/stage_projection_round_trip.rs
View file

@ -28,7 +28,10 @@ fn stage_projection_round_trips_representative_json() {
"parallel_results": [{ "branch": 0, "status": "succeeded" }],
"stdout": "ok",
"stderr": "",
"termination": "exited"
"termination": "exited",
"started_at": "2026-04-29T12:34:00Z",
"duration_ms": 56000,
"state": "succeeded"
});
let state: StageProjection = serde_json::from_value(value.clone()).unwrap();

12
lib/crates/fabro-cli/src/commands/auth/status.rs
View file

@ -1,5 +1,5 @@
use anyhow::Result;
use chrono::{DateTime, Utc};
use chrono::{DateTime, SecondsFormat, Utc};
use fabro_client::{AuthEntry, AuthStore, OAuthEntry};
use serde::Serialize;
@ -108,12 +108,12 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res
fabro_util::printerr!(
printer,
" Access expires: {}",
access_token_expires_at.to_rfc3339()
access_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true)
);
fabro_util::printerr!(
printer,
" Refresh expires: {}",
refresh_token_expires_at.to_rfc3339()
refresh_token_expires_at.to_rfc3339_opts(SecondsFormat::Secs, true)
);
}
StatusRow::DevToken {
@ -122,7 +122,11 @@ pub(super) fn status_command(args: &AuthStatusArgs, ctx: &CommandContext) -> Res
} => {
fabro_util::printerr!(printer, "{server}");
fabro_util::printerr!(printer, " Auth: dev-token");
fabro_util::printerr!(printer, " Logged in: {}", logged_in_at.to_rfc3339());
fabro_util::printerr!(
printer,
" Logged in: {}",
logged_in_at.to_rfc3339_opts(SecondsFormat::Secs, true)
);
}
}
}

51
lib/crates/fabro-cli/src/commands/run/logs.rs
View file

@ -14,7 +14,7 @@ use std::time::Duration;
use anyhow::{Context, Result, bail};
use chrono::{DateTime, Utc};
use fabro_redact::redact_jsonl_line;
use fabro_types::run_event::is_metadata_snapshot_compat_notice_code;
use fabro_types::RunNoticeCode;
use fabro_util::json::normalize_json_value;
use fabro_util::terminal::Styles;
use tokio::time;
@ -801,7 +801,9 @@ fn format_event_pretty_value(envelope: &serde_json::Value, styles: &Styles) -> O
}
fn is_metadata_snapshot_compat_notice(envelope: &serde_json::Value) -> bool {
prop_str_field(envelope, "code").is_some_and(is_metadata_snapshot_compat_notice_code)
prop_str_field(envelope, "code")
.and_then(|code| code.parse::<RunNoticeCode>().ok())
.is_some_and(RunNoticeCode::is_metadata_snapshot_compat)
}
fn str_field<'a>(value: &'a serde_json::Value, key: &str) -> Option<&'a str> {
@ -1150,14 +1152,27 @@ mod tests {
#[test]
fn pretty_run_notice_warn() {
let styles = no_color_styles();
let line = r#"{"ts":"2026-01-01T14:25:00Z","event":"run.notice","properties":{"level":"warn","code":"sandbox_cleanup_failed","message":"sandbox cleanup failed: boom"}}"#;
let result = format_event_pretty(line, &styles).unwrap();
let code = RunNoticeCode::SandboxCleanupFailed.to_string();
let line = serde_json::json!({
"ts": "2026-01-01T14:25:00Z",
"event": "run.notice",
"properties": {
"level": "warn",
"code": code,
"message": "sandbox cleanup failed: boom",
},
})
.to_string();
let result = format_event_pretty(&line, &styles).unwrap();
assert!(result.contains("Warning:"), "got: {result}");
assert!(
result.contains("sandbox cleanup failed: boom"),
"got: {result}"
);
assert!(result.contains("[sandbox_cleanup_failed]"), "got: {result}");
assert!(
result.contains(&format!("[{}]", RunNoticeCode::SandboxCleanupFailed)),
"got: {result}"
);
}
#[test]
@ -1235,13 +1250,31 @@ mod tests {
fn pretty_stream_suppresses_metadata_compat_notice_only() {
let styles = no_color_styles();
let failed = r#"{"ts":"2026-01-01T14:25:00Z","event":"metadata.snapshot.failed","properties":{"phase":"checkpoint","branch":"fabro/meta","duration_ms":900,"failure_kind":"write","error":"write failed"}}"#;
let compat_notice = r#"{"ts":"2026-01-01T14:25:01Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_write_failed","message":"legacy metadata warning"}}"#;
let degraded_notice = r#"{"ts":"2026-01-01T14:25:02Z","event":"run.notice","properties":{"level":"warn","code":"checkpoint_metadata_degraded","message":"metadata snapshots disabled"}}"#;
let compat_notice = serde_json::json!({
"ts": "2026-01-01T14:25:01Z",
"event": "run.notice",
"properties": {
"level": "warn",
"code": RunNoticeCode::CheckpointMetadataWriteFailed,
"message": "legacy metadata warning",
},
})
.to_string();
let degraded_notice = serde_json::json!({
"ts": "2026-01-01T14:25:02Z",
"event": "run.notice",
"properties": {
"level": "warn",
"code": RunNoticeCode::CheckpointMetadataDegraded,
"message": "metadata snapshots disabled",
},
})
.to_string();
let mut state = PrettyEventState::default();
assert!(format_event_pretty_streamed(failed, &styles, &mut state).is_some());
assert!(format_event_pretty_streamed(compat_notice, &styles, &mut state).is_none());
let degraded = format_event_pretty_streamed(degraded_notice, &styles, &mut state).unwrap();
assert!(format_event_pretty_streamed(&compat_notice, &styles, &mut state).is_none());
let degraded = format_event_pretty_streamed(&degraded_notice, &styles, &mut state).unwrap();
assert!(
degraded.contains("metadata snapshots disabled"),
"got: {degraded}"

7
lib/crates/fabro-cli/src/commands/run/run_progress/event.rs
View file

@ -527,7 +527,7 @@ fn display_value(value: &Value) -> Option<String> {
mod tests {
use fabro_agent::AgentEvent;
use fabro_types::{MetadataSnapshotFailureKind, MetadataSnapshotPhase, fixtures};
use fabro_workflow::event::{Event, to_run_event};
use fabro_workflow::event::{Event, RunNoticeCode, to_run_event};
use super::*;
@ -804,10 +804,11 @@ mod tests {
fn round_trip_run_notice() {
let event = Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "sandbox_cleanup_failed".into(),
code: RunNoticeCode::SandboxCleanupFailed.to_string(),
message: "sandbox cleanup failed".into(),
exec_output_tail: None,
};
let expected_code = RunNoticeCode::SandboxCleanupFailed.to_string();
let stored = to_run_event(&fixtures::RUN_1, &event);
let parsed = from_run_event(&stored).unwrap();
@ -817,7 +818,7 @@ mod tests {
level: RunNoticeLevel::Warn,
code,
message,
} if code == "sandbox_cleanup_failed" && message == "sandbox cleanup failed"
} if code == expected_code && message == "sandbox cleanup failed"
));
}

14
lib/crates/fabro-cli/src/commands/run/run_progress/mod.rs
View file

@ -3,8 +3,7 @@
reason = "sync CLI run-progress renderer: writes to std::io::stderr directly"
)]
use fabro_types::RunEvent;
use fabro_types::run_event::is_metadata_snapshot_compat_notice_code;
use fabro_types::{RunEvent, RunNoticeCode};
mod event;
mod info_display;
@ -444,7 +443,10 @@ impl ProgressUI {
message,
} => {
if self.saw_metadata_snapshot_failure
&& is_metadata_snapshot_compat_notice_code(&code)
&& code
.parse::<RunNoticeCode>()
.ok()
.is_some_and(RunNoticeCode::is_metadata_snapshot_compat)
{
return;
}
@ -1208,7 +1210,7 @@ mod tests {
emit(&mut ui, Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "sandbox_cleanup_failed".into(),
code: RunNoticeCode::SandboxCleanupFailed.to_string(),
message: "sandbox cleanup failed".into(),
exec_output_tail: None,
});
@ -1279,13 +1281,13 @@ mod tests {
});
emit(&mut ui, Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "checkpoint_metadata_write_failed".into(),
code: RunNoticeCode::CheckpointMetadataWriteFailed.to_string(),
message: "legacy metadata warning".into(),
exec_output_tail: None,
});
emit(&mut ui, Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "checkpoint_metadata_degraded".into(),
code: RunNoticeCode::CheckpointMetadataDegraded.to_string(),
message: "metadata snapshots are disabled for this run".into(),
exec_output_tail: None,
});

42
lib/crates/fabro-cli/src/commands/run/runner.rs
View file

@ -8,7 +8,6 @@ use std::collections::HashMap;
use std::io::{BufRead as StdBufRead, BufReader as StdBufReader};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use anyhow::{Context, Result, anyhow};
@ -34,6 +33,7 @@ use fabro_workflow::runtime_store::{RunStoreBackend, RunStoreHandle};
use tokio::signal::unix::{SignalKind, signal};
use tokio::sync::{Mutex, RwLock as AsyncRwLock, mpsc};
use tokio::time::sleep;
use tokio_util::sync::CancellationToken;
use crate::args::RunWorkerMode;
use crate::server_client;
@ -86,16 +86,16 @@ pub(crate) async fn execute(
worker_token.to_owned(),
)));
let interviewer = Arc::new(ControlInterviewer::new());
let cancel_token = Arc::new(AtomicBool::new(false));
let cancel_token = CancellationToken::new();
let emitter = Arc::new(Emitter::new(run_id));
let steering_hub = Arc::new(fabro_workflow::SteeringHub::new(Arc::clone(&emitter)));
spawn_worker_control_stream(
Arc::clone(&interviewer),
Arc::clone(&cancel_token),
cancel_token.clone(),
Arc::clone(&steering_hub),
)?;
let run_control = RunControlState::new();
install_signal_handlers(Arc::clone(&run_control), Arc::clone(&cancel_token))?;
install_signal_handlers(Arc::clone(&run_control), cancel_token.clone())?;
let vault = load_worker_vault(storage_dir.as_deref())?;
let github_app = {
let vault_guard = match &vault {
@ -106,7 +106,7 @@ pub(crate) async fn execute(
};
let services = StartServices {
run_id,
cancel_token: Some(Arc::clone(&cancel_token)),
cancel_token: cancel_token.clone(),
emitter,
interviewer,
steering_hub,
@ -169,7 +169,7 @@ enum WorkerControlStreamEvent {
)]
fn spawn_worker_control_stream(
interviewer: Arc<ControlInterviewer>,
cancel_token: Arc<AtomicBool>,
cancel_token: CancellationToken,
steering_hub: Arc<fabro_workflow::SteeringHub>,
) -> Result<()> {
let (event_tx, event_rx) = mpsc::unbounded_channel();
@ -214,7 +214,7 @@ fn read_worker_control_stream_blocking<R>(
async fn handle_worker_control_stream_events(
interviewer: Arc<ControlInterviewer>,
cancel_token: Arc<AtomicBool>,
cancel_token: CancellationToken,
steering_hub: Arc<fabro_workflow::SteeringHub>,
mut event_rx: mpsc::UnboundedReceiver<WorkerControlStreamEvent>,
) {
@ -235,7 +235,7 @@ async fn handle_worker_control_stream_events(
async fn apply_worker_control_line(
interviewer: &ControlInterviewer,
cancel_token: &AtomicBool,
cancel_token: &CancellationToken,
steering_hub: &fabro_workflow::SteeringHub,
line: &str,
) {
@ -254,7 +254,7 @@ async fn apply_worker_control_line(
.await;
}
WorkerControlMessage::RunCancel => {
cancel_token.store(true, Ordering::SeqCst);
cancel_token.cancel();
interviewer.interrupt_all().await;
}
WorkerControlMessage::Steer { text, kind, actor } => {
@ -575,7 +575,7 @@ fn clone_sandbox_requires_github_credentials(provider: &str) -> bool {
fn install_signal_handlers(
run_control: Arc<RunControlState>,
cancel_token: Arc<AtomicBool>,
cancel_token: CancellationToken,
) -> Result<()> {
#[cfg(unix)]
{
@ -595,17 +595,17 @@ fn install_signal_handlers(
});
let mut terminate = signal(SignalKind::terminate())?;
let terminate_cancel = Arc::clone(&cancel_token);
let terminate_cancel = cancel_token.clone();
tokio::spawn(async move {
while terminate.recv().await.is_some() {
terminate_cancel.store(true, Ordering::SeqCst);
terminate_cancel.cancel();
}
});
let mut interrupt = signal(SignalKind::interrupt())?;
tokio::spawn(async move {
while interrupt.recv().await.is_some() {
cancel_token.store(true, Ordering::SeqCst);
cancel_token.cancel();
}
});
}
@ -620,7 +620,6 @@ fn install_signal_handlers(
)]
mod tests {
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use chrono::Utc;
use fabro_auth::{AuthCredential, AuthDetails};
@ -637,6 +636,7 @@ mod tests {
};
use fabro_vault::{SecretType, Vault};
use fabro_workflow::event::RunEventSink;
use tokio_util::sync::CancellationToken;
use super::{
WorkerControlStreamEvent, WorkerTitlePhase, apply_worker_control_line,
@ -842,7 +842,7 @@ mod tests {
#[tokio::test]
async fn worker_control_line_routes_answer_by_question_id() {
let interviewer = Arc::new(ControlInterviewer::new());
let cancel_token = Arc::new(AtomicBool::new(false));
let cancel_token = CancellationToken::new();
let mut question = Question::new("Approve?", QuestionType::YesNo);
question.id = "q-1".to_string();
let ask_interviewer = Arc::clone(&interviewer);
@ -859,13 +859,13 @@ mod tests {
let answer = answer_task.await.unwrap().answer;
assert_eq!(answer.value, AnswerValue::Yes);
assert!(!cancel_token.load(Ordering::SeqCst));
assert!(!cancel_token.is_cancelled());
}
#[tokio::test]
async fn worker_control_line_cancel_sets_cancel_token_and_interrupts_pending_interviews() {
let interviewer = Arc::new(ControlInterviewer::new());
let cancel_token = Arc::new(AtomicBool::new(false));
let cancel_token = CancellationToken::new();
let mut question = Question::new("Approve?", QuestionType::YesNo);
question.id = "q-1".to_string();
let ask_interviewer = Arc::clone(&interviewer);
@ -883,7 +883,7 @@ mod tests {
let answer = answer_task.await.unwrap().answer;
assert_eq!(answer.value, AnswerValue::Interrupted);
assert!(cancel_token.load(Ordering::SeqCst));
assert!(cancel_token.is_cancelled());
}
#[tokio::test]
@ -916,7 +916,7 @@ mod tests {
#[tokio::test]
async fn worker_control_event_loop_eof_interrupts_pending_interviews() {
let interviewer = Arc::new(ControlInterviewer::new());
let cancel_token = Arc::new(AtomicBool::new(false));
let cancel_token = CancellationToken::new();
let mut question = Question::new("Approve?", QuestionType::YesNo);
question.id = "q-1".to_string();
let ask_interviewer = Arc::clone(&interviewer);
@ -929,7 +929,7 @@ mod tests {
let hub = test_steering_hub();
handle_worker_control_stream_events(
Arc::clone(&interviewer),
Arc::clone(&cancel_token),
cancel_token.clone(),
hub,
event_rx,
)
@ -937,7 +937,7 @@ mod tests {
let answer = answer_task.await.unwrap().answer;
assert_eq!(answer.value, AnswerValue::Interrupted);
assert!(!cancel_token.load(Ordering::SeqCst));
assert!(!cancel_token.is_cancelled());
}
#[tokio::test]

17
lib/crates/fabro-cli/tests/it/cmd/attach.rs
View file

@ -160,6 +160,7 @@ fn attach_replays_completed_detached_run() {
----- stdout -----
----- stderr -----
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Run Tests [TIME]
@ -267,6 +268,7 @@ fn attach_before_completion_streams_to_finished_state() {
----- stdout -----
----- stderr -----
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [DURATION]
✓ wait [DURATION]
@ -699,6 +701,21 @@ fn attach_json_errors_without_prompting_for_human_input() {
"run_id": "[ULID]",
"ts": "[TIMESTAMP]"
},
{
"actor": {
"kind": "worker",
"run_id": "[ULID]"
},
"event": "run.notice",
"id": "[EVENT_ID]",
"properties": {
"code": "worktree_skipped_no_git",
"level": "warn",
"message": "Worktree mode `always` requested but no Git repository was found; running without a worktree."
},
"run_id": "[ULID]",
"ts": "[TIMESTAMP]"
},
{
"actor": {
"kind": "worker",

6
lib/crates/fabro-cli/tests/it/cmd/dump.rs
View file

@ -278,9 +278,9 @@ fn dump_exports_completed_run_snapshot() {
");
assert_snapshot!(dump_file_summary(&output_dir), @"
checkpoints/0013.json
checkpoints/0017.json
checkpoints/0021.json
checkpoints/0014.json
checkpoints/0018.json
checkpoints/0022.json
events.jsonl
graph.fabro
run.json

1
lib/crates/fabro-cli/tests/it/cmd/run.rs
View file

@ -684,6 +684,7 @@ fn dry_run_simple() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Run Tests [TIME]

5
lib/crates/fabro-cli/tests/it/workflow/dry_run_examples.rs
View file

@ -21,6 +21,7 @@ fn dry_run_branching() {
warning [node: implement]: Node 'implement' has goal_gate=true but no retry_target or fallback_retry_target (goal_gate_has_retry)
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Plan [TIME]
@ -57,6 +58,7 @@ fn dry_run_conditions() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [TIME]
✓ Decide [TIME]
@ -91,6 +93,7 @@ fn dry_run_parallel() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [TIME]
✓ Fork Work [TIME]
@ -126,6 +129,7 @@ fn dry_run_styled() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ start [TIME]
✓ Plan [TIME]
@ -161,6 +165,7 @@ fn dry_run_legacy_tool() {
Run: [ULID]
Web UI: http://localhost:3000/runs/[ULID]
Warning: Worktree mode `always` requested but no Git repository was found; running without a worktree. [worktree_skipped_no_git]
Sandbox: local (ready in [TIME])
✓ Start [TIME]
✓ Echo [TIME]

1
lib/crates/fabro-cli/tests/it/workflow/real_cli.rs
View file

@ -34,6 +34,7 @@ async fn run_real_cli_test(provider: Provider, model: &str) {
&emitter,
&env,
None,
tokio_util::sync::CancellationToken::new(),
)
.await
.unwrap_or_else(|_| panic!("CLI backend ({provider}/{model}) should succeed"));

50
lib/crates/fabro-core/src/executor.rs
View file

@ -1,5 +1,6 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
#[cfg(test)]
use std::sync::atomic::Ordering;
use std::time::Instant;
use tokio::time::sleep;
@ -18,7 +19,7 @@ use crate::state::ExecutionState;
#[derive(Default)]
pub struct ExecutorOptions {
pub cancel_token: Option<Arc<AtomicBool>>,
pub cancel_token: Option<CancellationToken>,
pub stall_token: Option<CancellationToken>,
pub max_node_visits: Option<usize>,
}
@ -58,7 +59,7 @@ impl<G: Graph + 'static> ExecutorBuilder<G> {
}
#[must_use]
pub fn cancel_token(mut self, token: Arc<AtomicBool>) -> Self {
pub fn cancel_token(mut self, token: CancellationToken) -> Self {
self.options.cancel_token = Some(token);
self
}
@ -95,7 +96,7 @@ impl<G: Graph + 'static> Executor<G> {
loop {
// Check cancellation
if let Some(ref token) = self.options.cancel_token {
if token.load(Ordering::Relaxed) {
if token.is_cancelled() {
state.cancelled = true;
let outcome = Outcome::fail("run cancelled");
self.lifecycle.on_run_end(&outcome, &state).await;
@ -500,7 +501,8 @@ mod tests {
#[tokio::test]
async fn executor_builder_sets_cancel_token() {
let token = Arc::new(AtomicBool::new(true)); // already cancelled
let token = CancellationToken::new();
token.cancel(); // already cancelled
let g = linear_graph(&["start", "end"]);
let state = ExecutionState::new(&g).unwrap();
let executor =
@ -511,6 +513,38 @@ mod tests {
assert!(matches!(result, Err(Error::Cancelled)));
}
#[tokio::test]
async fn executor_cancel_token_fired_during_run_returns_cancelled() {
// Cancel token fired by a handler during the first node; the executor
// checks cancellation at the next node boundary and returns Cancelled.
let token = CancellationToken::new();
let token_clone = token.clone();
struct CancellingHandler(CancellationToken);
#[async_trait]
impl NodeHandler<TestGraph> for CancellingHandler {
async fn execute(
&self,
_node: &TestNode,
_context: &Context,
_g: &TestGraph,
) -> Result<Outcome> {
self.0.cancel();
Ok(Outcome::success())
}
}
let g = linear_graph(&["start", "work", "end"]);
let state = ExecutionState::new(&g).unwrap();
let executor = ExecutorBuilder::new(
Arc::new(CancellingHandler(token_clone)) as Arc<dyn NodeHandler<TestGraph>>
)
.cancel_token(token)
.build();
let result = executor.run(&g, state).await;
assert!(matches!(result, Err(Error::Cancelled)));
}
// ---- Step 9: Terminal nodes, goal gates, visit limits ----
#[tokio::test]
@ -908,10 +942,10 @@ mod tests {
#[tokio::test]
async fn executor_cancellation_stops_run() {
let token = Arc::new(AtomicBool::new(false));
let token = CancellationToken::new();
let token_clone = token.clone();
struct CancellingHandler(Arc<AtomicBool>);
struct CancellingHandler(CancellationToken);
#[async_trait]
impl NodeHandler<TestGraph> for CancellingHandler {
async fn execute(
@ -921,7 +955,7 @@ mod tests {
_g: &TestGraph,
) -> Result<Outcome> {
// Cancel after first node
self.0.store(true, Ordering::Relaxed);
self.0.cancel();
Ok(Outcome::success())
}
}

37
lib/crates/fabro-core/src/stall.rs
View file

@ -5,6 +5,7 @@ use std::time::Duration;
use tokio::sync::Notify;
use tokio::task::JoinHandle;
use tokio::time::sleep;
use tokio_util::sync::CancellationToken;
/// Trait for receiving stall timeout notifications.
pub trait ActivityMonitor: Send + Sync {
@ -16,11 +17,11 @@ pub trait ActivityMonitor: Send + Sync {
/// Watches for inactivity and fires a stall timeout if no activity is
/// reported within the configured duration.
pub struct StallWatchdog {
timeout: Duration,
cancel_token: Arc<AtomicBool>,
activity: Arc<Notify>,
shutdown: Arc<AtomicBool>,
monitor: Arc<dyn ActivityMonitor>,
timeout: Duration,
stall_token: CancellationToken,
activity: Arc<Notify>,
shutdown: Arc<AtomicBool>,
monitor: Arc<dyn ActivityMonitor>,
}
/// Guard that resets the stall timer on activity. Drop to stop watching.
@ -33,12 +34,12 @@ pub struct StallGuard {
impl StallWatchdog {
pub fn new(
timeout: Duration,
cancel_token: Arc<AtomicBool>,
stall_token: CancellationToken,
monitor: Arc<dyn ActivityMonitor>,
) -> Self {
Self {
timeout,
cancel_token,
stall_token,
activity: Arc::new(Notify::new()),
shutdown: Arc::new(AtomicBool::new(false)),
monitor,
@ -51,7 +52,7 @@ impl StallWatchdog {
let activity = self.activity.clone();
let shutdown = self.shutdown.clone();
let timeout = self.timeout;
let cancel_token = self.cancel_token;
let stall_token = self.stall_token;
let monitor = self.monitor;
let handle = tokio::spawn(async move {
@ -66,7 +67,7 @@ impl StallWatchdog {
"Stall timeout: no activity detected"
);
monitor.on_stall_timeout(timeout);
cancel_token.store(true, Ordering::Relaxed);
stall_token.cancel();
return;
}
() = activity.notified() => {
@ -136,7 +137,7 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stall_watchdog_cancels_on_inactivity() {
let cancel = Arc::new(AtomicBool::new(false));
let cancel = CancellationToken::new();
let monitor = TestMonitor::new();
let watchdog =
StallWatchdog::new(Duration::from_millis(50), cancel.clone(), monitor.clone());
@ -145,13 +146,13 @@ mod tests {
// Wait for timeout to fire
sleep(Duration::from_millis(100)).await;
assert!(cancel.load(Ordering::Relaxed));
assert!(cancel.is_cancelled());
assert_eq!(monitor.stalls(), 1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stall_watchdog_resets_on_activity() {
let cancel = Arc::new(AtomicBool::new(false));
let cancel = CancellationToken::new();
let monitor = TestMonitor::new();
let watchdog =
StallWatchdog::new(Duration::from_millis(80), cancel.clone(), monitor.clone());
@ -164,17 +165,17 @@ mod tests {
// After another 50ms (100ms total, but only 50ms since activity), should not
// have timed out
sleep(Duration::from_millis(50)).await;
assert!(!cancel.load(Ordering::Relaxed));
assert!(!cancel.is_cancelled());
// Wait long enough for timeout after last activity (80ms + margin)
sleep(Duration::from_millis(60)).await;
assert!(cancel.load(Ordering::Relaxed));
assert!(cancel.is_cancelled());
assert_eq!(monitor.stalls(), 1);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stall_watchdog_clean_shutdown_on_success() {
let cancel = Arc::new(AtomicBool::new(false));
let cancel = CancellationToken::new();
let monitor = TestMonitor::new();
let watchdog =
StallWatchdog::new(Duration::from_millis(50), cancel.clone(), monitor.clone());
@ -187,13 +188,13 @@ mod tests {
sleep(Duration::from_millis(100)).await;
// Should NOT have triggered
assert!(!cancel.load(Ordering::Relaxed));
assert!(!cancel.is_cancelled());
assert_eq!(monitor.stalls(), 0);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stall_guard_cleanup_on_drop() {
let cancel = Arc::new(AtomicBool::new(false));
let cancel = CancellationToken::new();
let monitor = TestMonitor::new();
let watchdog =
StallWatchdog::new(Duration::from_millis(50), cancel.clone(), monitor.clone());
@ -206,6 +207,6 @@ mod tests {
sleep(Duration::from_millis(150)).await;
// Cancel should not be set
assert!(!cancel.load(Ordering::Relaxed));
assert!(!cancel.is_cancelled());
}
}

7
lib/crates/fabro-dump/src/lib.rs
View file

@ -66,18 +66,13 @@ impl RunDump {
entries.push(RunDumpEntry::text("graph.fabro", graph_source.clone()));
}
let mut stages: Vec<_> = state.iter_stages().collect();
let stages: Vec<_> = state.iter_stages().collect();
if stages.len() > MAX_STAGES_IN_DUMP {
bail!(
"run dump supports at most {MAX_STAGES_IN_DUMP} stages with the current path prefix width (got {})",
stages.len()
);
}
stages.sort_by(|(left_id, left), (right_id, right)| {
left.first_event_seq
.cmp(&right.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
let mut stage_ranks = HashMap::new();
for (index, (stage_id, _)) in stages.iter().enumerate() {

5
lib/crates/fabro-retro/src/retro_agent.rs
View file

@ -204,7 +204,10 @@ pub async fn run_retro_agent(
// Optionally forward agent events via the callback
let event_forwarder_handle = event_callback.map(|cb| spawn_retro_event_forwarder(&session, cb));
session.initialize().await;
session
.initialize()
.await
.context("Retro agent session initialization failed")?;
let prompt = build_retro_prompt(RETRO_DATA_DIR);

51
lib/crates/fabro-sandbox/src/daytona/mod.rs
View file

@ -26,7 +26,7 @@ use tokio_util::sync::CancellationToken;
use crate::clone_source::{self, CloneDecision, EmptyWorkspaceReason};
use crate::redact::redact_auth_url;
use crate::sandbox::resolve_path;
use crate::sandbox::{optional_timeout, resolve_path};
use crate::{
CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, Sandbox,
SandboxEvent, SandboxEventCallback, format_lines_numbered, shell_quote,
@ -37,6 +37,9 @@ const DEFAULT_SNAPSHOT: &str = "daytona-medium";
pub const DEFAULT_DAYTONA_API_URL: &str = "https://app.daytona.io/api";
const FABRO_SANDBOX_USER_AGENT: &str = concat!("fabro-sandbox/", env!("CARGO_PKG_VERSION"));
const DAYTONA_PROBE_TIMEOUT: Duration = Duration::from_secs(20);
/// Upper bound on `DaytonaSession::close` so a stalled Daytona REST call cannot
/// block cancellation/timeout paths from returning.
const DAYTONA_SESSION_CLOSE_TIMEOUT: Duration = Duration::from_secs(10);
/// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow.
pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[
@ -1307,7 +1310,7 @@ impl Sandbox for DaytonaSandbox {
async fn exec_command_streaming(
&self,
command: &str,
timeout_ms: u64,
timeout_ms: Option<u64>,
working_dir: Option<&str>,
env_vars: Option<&HashMap<String, String>>,
cancel_token: Option<CancellationToken>,
@ -1397,7 +1400,7 @@ impl Sandbox for DaytonaSandbox {
&session,
&command_id,
session_exec.exit_code,
Duration::from_millis(timeout_ms),
timeout_ms,
cancel_token.unwrap_or_default(),
&mut stream_task,
)
@ -1673,19 +1676,39 @@ impl DaytonaSession {
}
/// Idempotent: a second call after `active=false` is a no-op.
///
/// `delete_session` is bounded by [`DAYTONA_SESSION_CLOSE_TIMEOUT`] so a
/// stalled Daytona REST call cannot block cancellation paths indefinitely.
async fn close(&mut self, reason: &'static str) {
if !self.active {
return;
}
self.active = false;
if let Some(svc) = self.process_svc.take() {
if let Err(err) = svc.delete_session(&self.session_id).await {
tracing::warn!(
error = %err,
session_id = %self.session_id,
reason,
"failed to delete Daytona session"
);
match time::timeout(
DAYTONA_SESSION_CLOSE_TIMEOUT,
svc.delete_session(&self.session_id),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(err)) => {
tracing::warn!(
error = %err,
session_id = %self.session_id,
reason,
"failed to delete Daytona session"
);
}
Err(_) => {
tracing::warn!(
session_id = %self.session_id,
reason,
timeout_ms = u64::try_from(DAYTONA_SESSION_CLOSE_TIMEOUT.as_millis())
.unwrap_or(u64::MAX),
"timed out deleting Daytona session"
);
}
}
}
}
@ -1730,7 +1753,7 @@ async fn wait_for_completion(
session: &DaytonaSession,
command_id: &str,
initial_exit_code: Option<i32>,
timeout: Duration,
timeout_ms: Option<u64>,
cancel_token: CancellationToken,
stream_task: &mut JoinHandle<Result<(), DaytonaError>>,
) -> crate::Result<WaitOutcome> {
@ -1742,8 +1765,8 @@ async fn wait_for_completion(
});
}
let timeout_sleep = time::sleep(timeout);
tokio::pin!(timeout_sleep);
let timeout_future = optional_timeout(timeout_ms);
tokio::pin!(timeout_future);
loop {
tokio::select! {
() = time::sleep(Duration::from_millis(250)) => {
@ -1765,7 +1788,7 @@ async fn wait_for_completion(
});
}
}
() = &mut timeout_sleep => {
() = &mut timeout_future => {
return Ok(WaitOutcome {
exit_code: None,
termination: CommandTermination::TimedOut,

14
lib/crates/fabro-sandbox/src/docker.rs
View file

@ -2,7 +2,7 @@ use std::collections::HashMap;
use std::fmt::Write as _;
use std::io::Cursor;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use std::time::{Instant, SystemTime, UNIX_EPOCH};
use async_trait::async_trait;
use bollard::Docker;
@ -24,7 +24,7 @@ use tokio_util::sync::CancellationToken;
use crate::clone_source::{self, CloneDecision, EmptyWorkspaceReason};
use crate::redact::redact_auth_url;
use crate::sandbox::resolve_path;
use crate::sandbox::{optional_timeout, resolve_path};
use crate::{
CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, Sandbox,
SandboxEvent, SandboxEventCallback, format_lines_numbered, shell_quote,
@ -362,7 +362,7 @@ impl DockerSandbox {
async fn docker_exec_shell_streaming(
&self,
command: &str,
timeout_ms: u64,
timeout_ms: Option<u64>,
working_dir: Option<&str>,
env_vars: Option<&HashMap<String, String>>,
cancel_token: Option<CancellationToken>,
@ -380,7 +380,8 @@ impl DockerSandbox {
controlled_command,
];
let timeout_duration = Duration::from_millis(timeout_ms);
let timeout_future = optional_timeout(timeout_ms);
tokio::pin!(timeout_future);
let token = cancel_token.unwrap_or_default();
let container_id = self.container_id()?.to_string();
@ -399,7 +400,7 @@ impl DockerSandbox {
joined
.map_err(|e| crate::Error::context("Docker exec stream task failed", e))??
}
() = time::sleep(timeout_duration) => {
() = &mut timeout_future => {
termination = CommandTermination::TimedOut;
self.request_docker_exec_stop(&stop_file).await?;
output_task
@ -1192,7 +1193,7 @@ impl Sandbox for DockerSandbox {
async fn exec_command_streaming(
&self,
command: &str,
timeout_ms: u64,
timeout_ms: Option<u64>,
working_dir: Option<&str>,
env_vars: Option<&HashMap<String, String>>,
cancel_token: Option<CancellationToken>,
@ -1539,6 +1540,7 @@ mod tests {
reason = "unit test reads an in-memory tar entry synchronously"
)]
use std::io::Read as _;
use std::time::Duration;
use tokio::process::Command;

67
lib/crates/fabro-sandbox/src/local.rs
View file

@ -10,6 +10,7 @@ use tokio::task::spawn_blocking;
use tokio::{fs, time};
use tokio_util::sync::CancellationToken;
use crate::sandbox::optional_timeout;
use crate::{
CommandOutputCallback, DirEntry, ExecResult, ExecStreamingResult, GrepOptions, Sandbox,
SandboxEvent, SandboxEventCallback, format_lines_numbered,
@ -126,6 +127,19 @@ fn process_env_vars() -> Vec<(String, String)> {
std::env::vars().collect()
}
async fn drain_pipe<R>(mut pipe: Option<R>, stream: CommandOutputStream) -> String
where
R: AsyncRead + Unpin,
{
let mut buf = String::new();
if let Some(ref mut reader) = pipe {
if let Err(err) = reader.read_to_string(&mut buf).await {
tracing::warn!(error = %err, ?stream, "Failed to drain child output");
}
}
buf
}
#[async_trait]
impl Sandbox for LocalSandbox {
async fn read_file(
@ -277,22 +291,12 @@ impl Sandbox for LocalSandbox {
// it writes more than the OS pipe buffer (~64 KB) the write() syscall
// blocks until the parent drains the pipe, but the parent is blocked
// on child.wait().
let mut stdout_pipe = child.stdout.take();
let mut stderr_pipe = child.stderr.take();
let stdout_task = tokio::spawn(async move {
let mut buf = String::new();
if let Some(ref mut r) = stdout_pipe {
let _ = r.read_to_string(&mut buf).await;
}
buf
});
let stderr_task = tokio::spawn(async move {
let mut buf = String::new();
if let Some(ref mut r) = stderr_pipe {
let _ = r.read_to_string(&mut buf).await;
}
buf
});
let stdout_pipe = child.stdout.take();
let stderr_pipe = child.stderr.take();
let stdout_task =
tokio::spawn(async move { drain_pipe(stdout_pipe, CommandOutputStream::Stdout).await });
let stderr_task =
tokio::spawn(async move { drain_pipe(stderr_pipe, CommandOutputStream::Stderr).await });
let (termination, exit_code) = tokio::select! {
status_result = child.wait() => {
@ -327,7 +331,7 @@ impl Sandbox for LocalSandbox {
async fn exec_command_streaming(
&self,
command: &str,
timeout_ms: u64,
timeout_ms: Option<u64>,
working_dir: Option<&str>,
env_vars: Option<&std::collections::HashMap<String, String>>,
cancel_token: Option<CancellationToken>,
@ -367,7 +371,8 @@ impl Sandbox for LocalSandbox {
.spawn()
.map_err(|e| crate::Error::context("Failed to spawn command", e))?;
let timeout_duration = std::time::Duration::from_millis(timeout_ms);
let timeout_future = optional_timeout(timeout_ms);
tokio::pin!(timeout_future);
let token = cancel_token.unwrap_or_default();
let stdout_pipe = child.stdout.take();
@ -387,7 +392,7 @@ impl Sandbox for LocalSandbox {
.map_err(|e| crate::Error::context("Failed to wait for process", e))?;
(CommandTermination::Exited, status.code())
}
() = time::sleep(timeout_duration) => {
() = &mut timeout_future => {
sigterm_then_kill(&mut child).await;
(CommandTermination::TimedOut, None)
}
@ -712,7 +717,12 @@ where
)]
mod tests {
use std::collections::HashMap;
use std::io;
use std::path::PathBuf;
use std::pin::Pin;
use std::task::{Context as TaskContext, Poll};
use tokio::io::ReadBuf;
use super::*;
@ -722,6 +732,25 @@ mod tests {
dir
}
#[tokio::test]
async fn drain_pipe_returns_empty_buffer_after_read_failure() {
struct FailingReader;
impl AsyncRead for FailingReader {
fn poll_read(
self: Pin<&mut Self>,
_cx: &mut TaskContext<'_>,
_buf: &mut ReadBuf<'_>,
) -> Poll<io::Result<()>> {
Poll::Ready(Err(io::Error::other("simulated read failure")))
}
}
let output = drain_pipe(Some(FailingReader), CommandOutputStream::Stdout).await;
assert!(output.is_empty());
}
#[tokio::test]
async fn read_file_with_line_numbers() {
let dir = temp_dir();

23
lib/crates/fabro-sandbox/src/sandbox.rs
View file

@ -17,6 +17,16 @@ const GIT: &str = "git -c maintenance.auto=0 -c gc.auto=0";
pub const DEFAULT_EXEC_OUTPUT_TAIL_BYTES: usize = 8 * 1024;
/// Sleep for `timeout_ms` if `Some`, otherwise never resolves. Used by
/// streaming `exec_command` impls to model "no timeout" without scheduling a
/// `Duration::from_millis(u64::MAX)` sleep.
pub(crate) async fn optional_timeout(timeout_ms: Option<u64>) {
match timeout_ms {
Some(ms) => time::sleep(Duration::from_millis(ms)).await,
None => std::future::pending::<()>().await,
}
}
/// Information returned when a sandbox sets up git for a workflow run.
#[derive(Debug, Clone)]
pub struct GitRunInfo {
@ -93,7 +103,7 @@ macro_rules! delegate_sandbox {
async fn exec_command_streaming(
&self,
command: &str,
timeout_ms: u64,
timeout_ms: Option<u64>,
working_dir: Option<&str>,
env_vars: Option<&std::collections::HashMap<String, String>>,
cancel_token: Option<tokio_util::sync::CancellationToken>,
@ -607,14 +617,21 @@ pub trait Sandbox: Send + Sync {
async fn exec_command_streaming(
&self,
command: &str,
timeout_ms: u64,
timeout_ms: Option<u64>,
working_dir: Option<&str>,
env_vars: Option<&std::collections::HashMap<String, String>>,
cancel_token: Option<CancellationToken>,
output_callback: CommandOutputCallback,
) -> crate::Result<ExecStreamingResult> {
let fallback_timeout_ms = timeout_ms.unwrap_or(u64::MAX);
let result = self
.exec_command(command, timeout_ms, working_dir, env_vars, cancel_token)
.exec_command(
command,
fallback_timeout_ms,
working_dir,
env_vars,
cancel_token,
)
.await?;
if !result.stdout.is_empty() {
output_callback(

2
lib/crates/fabro-sandbox/src/worktree.rs
View file

@ -236,7 +236,7 @@ impl Sandbox for WorktreeSandbox {
async fn exec_command_streaming(
&self,
command: &str,
timeout_ms: u64,
timeout_ms: Option<u64>,
working_dir: Option<&str>,
env_vars: Option<&HashMap<String, String>>,
cancel_token: Option<CancellationToken>,

11
lib/crates/fabro-sandbox/tests/daytona_streaming_live.rs
View file

@ -63,7 +63,7 @@ mod daytona_streaming_live {
sandbox_for_exec
.exec_command_streaming(
"printf 'live-out\\n'; printf 'live-err\\n' >&2; sleep 30",
60_000,
Some(60_000),
None,
None,
Some(cancel_for_exec),
@ -186,7 +186,14 @@ mod daytona_streaming_live {
let chunks = Arc::new(Mutex::new(Vec::new()));
let callback = capture_callback(Arc::clone(&chunks));
let result = sandbox
.exec_command_streaming(command, timeout_ms, None, None, cancel_token, callback)
.exec_command_streaming(
command,
Some(timeout_ms),
None,
None,
cancel_token,
callback,
)
.await?;
let chunks = chunks.lock().await.clone();

2
lib/crates/fabro-sandbox/tests/docker_streaming.rs
View file

@ -49,7 +49,7 @@ async fn streaming_timeout_terminates_docker_exec_before_returning() {
let result = sandbox
.exec_command_streaming(
&format!("trap '' HUP TERM; echo start; sleep 5 # {marker}"),
200,
Some(200),
None,
None,
None,

3
lib/crates/fabro-server/Cargo.toml
View file

@ -56,6 +56,7 @@ globset.workspace = true
tower = "0.5"
tower-http = { version = "0.6", features = ["trace"] }
tokio-stream = { workspace = true, features = ["sync"] }
tokio-util.workspace = true
base64.workspace = true
jsonwebtoken.workspace = true
hkdf.workspace = true
@ -108,4 +109,4 @@ tokio-util.workspace = true
fabro-macros = { path = "../fabro-macros" }
fabro-sandbox = { path = "../fabro-sandbox", features = ["test-support"] }
fabro-test = { workspace = true }
fabro-types = { path = "../fabro-types", features = ["test-support"] }
fabro-types = { path = "../fabro-types", features = ["test-support"] }

273
lib/crates/fabro-server/src/demo/mod.rs
View file

@ -15,15 +15,16 @@ use axum::http::StatusCode;
use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response};
use fabro_api::types::{
CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, FileDiff, FileDiffChangeKind,
PaginatedRunFileList, RunArtifactListResponse, RunFilesMeta,
CreateSecretRequest, DeleteSecretRequest, DiffFile, DiffStats, EventEnvelope, FileDiff,
FileDiffChangeKind, PaginatedEventList, PaginatedRunFileList, PaginationMeta,
RunArtifactListResponse, RunFilesMeta,
};
use serde_json::json;
use crate::error::ApiError;
use crate::principal_middleware::RequiredUser;
use crate::run_selector::{ResolveRunError, resolve_run_by_selector};
use crate::server::{AppState, PaginationParams};
use crate::server::{AppState, EventListParams, PaginationParams, parse_stage_id_path};
fn paginated_response<T: serde::Serialize>(
items: Vec<T>,
@ -133,13 +134,39 @@ pub(crate) async fn get_run_stages(
paginated_response(runs::stages(), &pagination)
}
pub(crate) async fn get_stage_turns(
pub(crate) async fn get_stage_events(
_auth: RequiredUser,
State(_state): State<Arc<AppState>>,
Path((_id, _stage_id)): Path<(String, String)>,
Query(pagination): Query<PaginationParams>,
Path((_id, stage_id)): Path<(String, String)>,
Query(params): Query<EventListParams>,
) -> Response {
paginated_response(runs::turns(), &pagination)
let stage_id = match parse_stage_id_path(&stage_id) {
Ok(stage_id) => stage_id,
Err(response) => return response,
};
let since_seq = params.since_seq();
let limit = params.limit();
let mut matches: Vec<EventEnvelope> = runs::stage_events()
.into_iter()
.filter(|envelope| {
envelope.seq >= since_seq
&& (envelope.event.stage_id.as_ref() == Some(&stage_id)
|| (envelope.event.stage_id.is_none()
&& stage_id.visit() == 1
&& envelope.event.node_id.as_deref() == Some(stage_id.node_id())))
})
.take(limit + 1)
.collect();
let has_more = matches.len() > limit;
matches.truncate(limit);
(
StatusCode::OK,
Json(PaginatedEventList {
data: matches,
meta: PaginationMeta { has_more },
}),
)
.into_response()
}
pub(crate) async fn list_run_artifacts_stub(
@ -784,9 +811,10 @@ mod runs {
RunNamespace, RunPrepareSettings, RunSandboxSettings,
};
use fabro_types::settings::{InterpString, ProjectNamespace, WorkflowNamespace};
use fabro_types::{RunId, WorkflowSettings};
use fabro_types::{RunId, StageId, WorkflowSettings};
use super::ts;
use crate::server::run_stage_from_stage_id;
fn labels(entries: &[(&str, &str)]) -> HashMap<String, String> {
entries
@ -795,8 +823,8 @@ mod runs {
.collect()
}
fn demo_run_ids() -> &'static [RunId; 6] {
static IDS: OnceLock<[RunId; 6]> = OnceLock::new();
fn demo_run_ids() -> &'static [RunId; 7] {
static IDS: OnceLock<[RunId; 7]> = OnceLock::new();
IDS.get_or_init(|| {
[
RunId::with_timestamp(ts("2026-03-06T14:30:00Z"), 1),
@ -805,6 +833,7 @@ mod runs {
RunId::with_timestamp(ts("2026-03-04T10:00:00Z"), 4),
RunId::with_timestamp(ts("2026-03-03T16:45:00Z"), 5),
RunId::with_timestamp(ts("2026-02-28T14:00:00Z"), 6),
RunId::with_timestamp(ts("2026-03-06T14:35:00Z"), 7),
]
})
}
@ -974,23 +1003,27 @@ mod runs {
pub(super) fn columns() -> Vec<BoardColumnDefinition> {
vec![
BoardColumnDefinition {
id: "initializing".into(),
id: BoardColumn::Queued,
name: "Queued".into(),
},
BoardColumnDefinition {
id: BoardColumn::Initializing,
name: "Initializing".into(),
},
BoardColumnDefinition {
id: "running".into(),
id: BoardColumn::Running,
name: "Running".into(),
},
BoardColumnDefinition {
id: "blocked".into(),
id: BoardColumn::Blocked,
name: "Blocked".into(),
},
BoardColumnDefinition {
id: "succeeded".into(),
id: BoardColumn::Succeeded,
name: "Succeeded".into(),
},
BoardColumnDefinition {
id: "failed".into(),
id: BoardColumn::Failed,
name: "Failed".into(),
},
]
@ -1082,6 +1115,20 @@ mod runs {
Some(720000),
&[("release", "preview")],
),
summary(
7,
"api-server",
"implement",
"Implement",
"Add audit log retention policy",
"queued",
"2026-03-06T14:35:00Z",
None,
None,
None,
None,
&[("owner", "platform")],
),
]
}
@ -1150,54 +1197,164 @@ mod runs {
None,
None,
),
board_item(
take_summary(&mut summaries, demo_run_id(7)),
BoardColumn::Queued,
None,
None,
None,
),
]
}
pub(super) fn stages() -> Vec<RunStage> {
vec![
RunStage {
id: "detect-drift".into(),
name: "Detect Drift".into(),
status: StageState::Succeeded,
duration_secs: Some(72.0),
dot_id: Some("detect".into()),
},
RunStage {
id: "propose-changes".into(),
name: "Propose Changes".into(),
status: StageState::Succeeded,
duration_secs: Some(154.0),
dot_id: Some("propose".into()),
},
RunStage {
id: "review-changes".into(),
name: "Review Changes".into(),
status: StageState::Succeeded,
duration_secs: Some(45.0),
dot_id: Some("review".into()),
},
RunStage {
id: "apply-changes".into(),
name: "Apply Changes".into(),
status: StageState::Running,
duration_secs: Some(118.0),
dot_id: Some("apply".into()),
},
run_stage_from_stage_id(
&StageId::new("detect-drift", 1),
"Detect Drift",
StageState::Succeeded,
Some(72.0),
None,
),
run_stage_from_stage_id(
&StageId::new("propose-changes", 1),
"Propose Changes",
StageState::Succeeded,
Some(154.0),
None,
),
run_stage_from_stage_id(
&StageId::new("review-changes", 1),
"Review Changes",
StageState::Succeeded,
Some(45.0),
None,
),
run_stage_from_stage_id(
&StageId::new("apply-changes", 1),
"Apply Changes",
StageState::Succeeded,
Some(118.0),
None,
),
run_stage_from_stage_id(
&StageId::new("apply-changes", 2),
"Apply Changes",
StageState::Running,
None,
None,
),
]
}
pub(super) fn turns() -> Vec<StageTurn> {
pub(super) fn stage_events() -> Vec<fabro_types::EventEnvelope> {
use fabro_model::BilledTokenCounts;
use fabro_types::run_event::agent::{
AgentMessageProps, AgentToolCompletedProps, AgentToolStartedProps,
};
use fabro_types::run_event::stage::StagePromptProps;
use fabro_types::{EventBody, EventEnvelope, RunEvent};
let run_id = demo_run_id(1);
let node_id = "detect-drift";
let stage_id = fabro_types::StageId::new(node_id, 1);
let ts = ts("2026-03-06T14:30:00Z");
let make_envelope = |seq: u32, id: &str, body: EventBody| EventEnvelope {
seq,
event: RunEvent {
id: id.into(),
ts,
run_id,
node_id: Some(node_id.into()),
node_label: Some("Detect Drift".into()),
stage_id: Some(stage_id.clone()),
parallel_group_id: None,
parallel_branch_id: None,
session_id: None,
parent_session_id: None,
tool_call_id: None,
actor: None,
body,
},
};
vec![
StageTurn::SystemStageTurn(SystemStageTurn { kind: SystemStageTurnKind::System, content: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into() }),
StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I'll start by loading the environment configurations for both production and staging to compare them.".into() }),
StageTurn::ToolStageTurn(ToolStageTurn {
kind: ToolStageTurnKind::Tool, content: None,
tools: vec![
ToolUse { id: "toolu_01".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/production/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-prod.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(45) },
ToolUse { id: "toolu_02".into(), tool_name: "read_file".into(), input: r#"{ "path": "environments/staging/config.toml" }"#.into(), result: "[redis]\nhost = \"redis-staging.internal\"\nport = 6379".into(), is_error: false, duration_ms: Some(38) },
],
}),
StageTurn::AssistantStageTurn(AssistantStageTurn { kind: AssistantStageTurnKind::Assistant, content: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into() }),
make_envelope(
1,
"evt-detect-drift-1",
EventBody::StagePrompt(StagePromptProps {
visit: 1,
text: "You are a drift detection agent. Compare the production and staging environments and identify any configuration or code drift.".into(),
mode: None,
provider: None,
model: None,
}),
),
make_envelope(
2,
"evt-detect-drift-2",
EventBody::AgentMessage(AgentMessageProps {
text: "I'll start by loading the environment configurations for both production and staging to compare them.".into(),
model: "Opus 4.6".into(),
billing: BilledTokenCounts::default(),
tool_call_count: 0,
visit: 1,
}),
),
make_envelope(
3,
"evt-detect-drift-3",
EventBody::AgentToolStarted(AgentToolStartedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_01".into(),
arguments: serde_json::json!({ "path": "environments/production/config.toml" }),
visit: 1,
}),
),
make_envelope(
4,
"evt-detect-drift-4",
EventBody::AgentToolCompleted(AgentToolCompletedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_01".into(),
output: serde_json::json!("[redis]\nhost = \"redis-prod.internal\"\nport = 6379"),
is_error: false,
visit: 1,
}),
),
make_envelope(
5,
"evt-detect-drift-5",
EventBody::AgentToolStarted(AgentToolStartedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_02".into(),
arguments: serde_json::json!({ "path": "environments/staging/config.toml" }),
visit: 1,
}),
),
make_envelope(
6,
"evt-detect-drift-6",
EventBody::AgentToolCompleted(AgentToolCompletedProps {
tool_name: "read_file".into(),
tool_call_id: "toolu_02".into(),
output: serde_json::json!("[redis]\nhost = \"redis-staging.internal\"\nport = 6379"),
is_error: false,
visit: 1,
}),
),
make_envelope(
7,
"evt-detect-drift-7",
EventBody::AgentMessage(AgentMessageProps {
text: "I've detected drift in 3 resources between production and staging:\n\n1. **redis.max_connections** — production has 200, staging has 100\n2. **redis.tls** — enabled in production, disabled in staging\n3. **iam.session_duration** — production uses 3600s, staging uses 1800s".into(),
model: "Opus 4.6".into(),
billing: BilledTokenCounts::default(),
tool_call_count: 0,
visit: 1,
}),
),
]
}
@ -1222,6 +1379,8 @@ mod runs {
total_usd_micros: Some(480_000),
},
runtime_secs: 72.0,
started_at: None,
state: Some(StageState::Succeeded),
},
RunBillingStage {
stage: BillingStageRef {
@ -1241,6 +1400,8 @@ mod runs {
total_usd_micros: Some(720_000),
},
runtime_secs: 154.0,
started_at: None,
state: Some(StageState::Succeeded),
},
RunBillingStage {
stage: BillingStageRef {
@ -1260,6 +1421,8 @@ mod runs {
total_usd_micros: Some(190_000),
},
runtime_secs: 45.0,
started_at: None,
state: Some(StageState::Succeeded),
},
RunBillingStage {
stage: BillingStageRef {
@ -1279,6 +1442,8 @@ mod runs {
total_usd_micros: Some(870_000),
},
runtime_secs: 118.0,
started_at: None,
state: Some(StageState::Running),
},
],
totals: RunBillingTotals {

18
lib/crates/fabro-server/src/principal_middleware.rs
View file

@ -52,6 +52,7 @@ pub(crate) struct RequestAuth(pub(crate) AuthContextSlot);
pub(crate) struct RequiredUser(pub(crate) UserPrincipal);
pub(crate) struct RequireRunScoped(pub(crate) RunId);
pub(crate) struct RequireRunBlob(pub(crate) RunId, pub(crate) RunBlobId);
pub(crate) struct RequireRunStageScoped(pub(crate) RunId, pub(crate) String);
pub(crate) struct RequireStageArtifact(pub(crate) RunId, pub(crate) StageId);
pub(crate) struct RequireCommandLog(
pub(crate) RunId,
@ -203,6 +204,23 @@ impl FromRequestParts<Arc<AppState>> for RequireRunBlob {
}
}
impl FromRequestParts<Arc<AppState>> for RequireRunStageScoped {
type Rejection = Response;
async fn from_request_parts(
parts: &mut Parts,
state: &Arc<AppState>,
) -> Result<Self, Self::Rejection> {
let Path((id, stage_id)): Path<(String, String)> = Path::from_request_parts(parts, state)
.await
.map_err(IntoResponse::into_response)?;
let run_id = parse_run_id_path(&id)?;
require_worker_or_user_for_run(&auth_slot_from_parts(parts), &run_id)
.map_err(IntoResponse::into_response)?;
Ok(Self(run_id, stage_id))
}
}
impl FromRequestParts<Arc<AppState>> for RequireStageArtifact {
type Rejection = Response;

149
lib/crates/fabro-server/src/server.rs
View file

@ -54,7 +54,7 @@ use fabro_llm::types::{
ContentPart, FinishReason, Message as LlmMessage, Request as LlmRequest, Role, ToolChoice,
ToolDefinition,
};
use fabro_model::{BilledModelUsage, BilledTokenCounts, Catalog, ModelTestMode, Provider};
use fabro_model::{BilledTokenCounts, Catalog, ModelTestMode, Provider};
use fabro_redact::redact_jsonl_line;
use fabro_sandbox::daytona::{self, DaytonaSandbox};
use fabro_sandbox::reconnect::reconnect;
@ -111,6 +111,7 @@ use tokio::task::spawn_blocking;
use tokio::time::{sleep, timeout};
use tokio_stream::StreamExt;
use tokio_stream::wrappers::{BroadcastStream, UnboundedReceiverStream};
use tokio_util::sync::CancellationToken;
use tower::{ServiceExt, service_fn};
use tracing::{Instrument, debug, error, info, warn};
use ulid::Ulid;
@ -125,7 +126,7 @@ use crate::ip_allowlist::{IpAllowlistConfig, ip_allowlist_middleware};
use crate::jwt_auth::{self, AuthMode};
use crate::principal_middleware::{
AuthContextSlot, RequestAuth, RequestAuthContext, RequireRunBlob, RequireRunScoped,
RequireStageArtifact, RequiredUser, principal_middleware,
RequireRunStageScoped, RequireStageArtifact, RequiredUser, principal_middleware,
};
use crate::request_id::{self, RequestId};
use crate::run_files::{FilesInFlight, new_files_in_flight};
@ -138,6 +139,7 @@ use crate::{
mod handler;
pub(crate) use handler::events::EventListParams;
#[cfg(test)]
pub(in crate::server) use handler::events::filtered_global_events;
pub(crate) use handler::graph::render_graph_bytes;
@ -205,7 +207,7 @@ struct ManagedRun {
event_tx: Option<broadcast::Sender<RunEvent>>,
checkpoint: Option<Checkpoint>,
cancel_tx: Option<oneshot::Sender<()>>,
cancel_token: Option<Arc<AtomicBool>>,
cancel_token: Option<CancellationToken>,
worker_pid: Option<u32>,
worker_pgid: Option<u32>,
run_dir: Option<std::path::PathBuf>,
@ -568,17 +570,50 @@ pub(crate) struct ResolvedAppStateSettings {
pub(crate) manifest_run_settings: std::result::Result<RunNamespace, SharedError>,
}
fn accumulate_model_billing(entry: &mut ModelBillingTotals, usage: &BilledModelUsage) {
let tokens = usage.tokens();
entry.stages += 1;
entry.billing.input_tokens += tokens.input_tokens;
entry.billing.output_tokens += tokens.output_tokens;
entry.billing.reasoning_tokens += tokens.reasoning_tokens;
entry.billing.cache_read_tokens += tokens.cache_read_tokens;
entry.billing.cache_write_tokens += tokens.cache_write_tokens;
entry.billing.total_tokens += tokens.total_tokens();
if let Some(value) = usage.total_usd_micros {
*entry.billing.total_usd_micros.get_or_insert(0) += value;
fn accumulate_billed_token_counts(target: &mut BilledTokenCounts, source: &BilledTokenCounts) {
target.input_tokens += source.input_tokens;
target.output_tokens += source.output_tokens;
target.reasoning_tokens += source.reasoning_tokens;
target.cache_read_tokens += source.cache_read_tokens;
target.cache_write_tokens += source.cache_write_tokens;
target.total_tokens += source.total_tokens;
if let Some(value) = source.total_usd_micros {
*target.total_usd_micros.get_or_insert(0) += value;
}
}
fn accumulate_billing_rollup(
accumulator: &mut BillingAccumulator,
rollup: &fabro_workflow::ProjectionBillingRollup,
) {
accumulator.total_runs += 1;
accumulator.total_runtime_secs += rollup.runtime_ms as f64 / 1000.0;
for model in &rollup.by_model {
let entry = accumulator
.by_model
.entry(model.model_id.clone())
.or_default();
entry.stages += model.stages;
accumulate_billed_token_counts(&mut entry.billing, &model.billing);
}
}
pub(crate) fn run_stage_from_stage_id(
stage_id: &StageId,
name: impl Into<String>,
status: StageState,
duration_secs: Option<f64>,
started_at: Option<chrono::DateTime<chrono::Utc>>,
) -> RunStage {
RunStage {
id: stage_id.to_string(),
name: name.into(),
status,
duration_secs,
node_id: stage_id.node_id().to_string(),
visit: std::num::NonZeroU32::new(stage_id.visit())
.expect("StageId stores a non-zero visit"),
started_at,
}
}
@ -1520,7 +1555,7 @@ async fn delete_run_internal(
if let Some(mut managed_run) = managed_run {
if let Some(token) = &managed_run.cancel_token {
token.store(true, Ordering::SeqCst);
token.cancel();
}
if let Some(answer_transport) = managed_run.answer_transport.clone() {
let _ = answer_transport.cancel_run().await;
@ -2656,12 +2691,12 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
let (cancel_tx, cancel_rx) = oneshot::channel::<()>();
let cancel_token = Arc::new(AtomicBool::new(false));
let cancel_token = CancellationToken::new();
let (event_tx, _) = broadcast::channel(256);
managed_run.status = RunStatus::Starting;
managed_run.cancel_tx = Some(cancel_tx);
managed_run.cancel_token = Some(Arc::clone(&cancel_token));
managed_run.cancel_token = Some(cancel_token.clone());
managed_run.event_tx = Some(event_tx);
(
@ -2756,7 +2791,7 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
let server_settings = state.server_settings();
let github_settings = &server_settings.server.integrations.github;
if cancel_token.load(Ordering::SeqCst) {
if cancel_token.is_cancelled() {
finish_cancelled_run_before_execution(&state, run_id).await;
return;
}
@ -2791,7 +2826,7 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
let github_app = match github_app_result {
Ok(github_app) => github_app,
Err(e) => {
if cancel_token.load(Ordering::SeqCst) {
if cancel_token.is_cancelled() {
finish_cancelled_run_before_execution(&state, run_id).await;
return;
}
@ -2818,7 +2853,7 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
.collect();
let services = operations::StartServices {
run_id,
cancel_token: Some(Arc::clone(&cancel_token)),
cancel_token: cancel_token.clone(),
emitter: Arc::clone(&emitter),
interviewer: Arc::clone(&interview_runtime),
steering_hub: Arc::clone(&steering_hub),
@ -2843,7 +2878,7 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
let result = tokio::select! {
result = execution => ExecutionResult::Completed(Box::new(result)),
_ = cancel_rx => {
cancel_token.store(true, Ordering::SeqCst);
cancel_token.cancel();
ExecutionResult::CancelledBySignal
}
};
@ -2854,9 +2889,9 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
}
}
// Save final checkpoint
let checkpoint = match run_store.state().await {
Ok(state) => state.checkpoint,
// Save final projection
let final_projection = match run_store.state().await {
Ok(state) => Some(state),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run state from store");
None
@ -2864,32 +2899,17 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
// Accumulate aggregate usage after execution completes.
if let Some(ref cp) = checkpoint {
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store");
HashMap::default()
}
};
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
agg.total_runs += 1;
let mut run_runtime: f64 = 0.0;
for (node_id, outcome) in &cp.node_outcomes {
if let Some(usage) = &outcome.usage {
let entry = agg
.by_model
.entry(usage.model_id().to_string())
.or_default();
accumulate_model_billing(entry, usage);
}
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
run_runtime += duration_ms as f64 / 1000.0;
if let Some(ref projection) = final_projection {
if projection.checkpoint.is_some() {
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
accumulate_billing_rollup(
&mut agg,
&fabro_workflow::billing_rollup_from_projection(projection),
);
}
agg.total_runtime_secs += run_runtime;
}
let mut runs = state.runs.lock().expect("runs lock poisoned");
@ -2938,7 +2958,9 @@ async fn execute_run_in_process(state: Arc<AppState>, run_id: RunId) {
};
}
}
managed_run.checkpoint = checkpoint;
managed_run.checkpoint = final_projection
.as_ref()
.and_then(|projection| projection.checkpoint.clone());
managed_run.run_dir = Some(run_dir);
clear_live_run_state(managed_run);
}
@ -3181,32 +3203,15 @@ async fn execute_run_subprocess(state: Arc<AppState>, run_id: RunId) {
}
};
if let Some(ref checkpoint) = final_state.checkpoint {
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
tracing::warn!(run_id = %run_id, error = %err, "Failed to load run events from store");
HashMap::default()
}
};
if final_state.checkpoint.is_some() {
let mut agg = state
.aggregate_billing
.lock()
.expect("aggregate_billing lock poisoned");
agg.total_runs += 1;
let mut run_runtime: f64 = 0.0;
for (node_id, outcome) in &checkpoint.node_outcomes {
if let Some(usage) = &outcome.usage {
let entry = agg
.by_model
.entry(usage.model_id().to_string())
.or_default();
accumulate_model_billing(entry, usage);
}
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
run_runtime += duration_ms as f64 / 1000.0;
}
agg.total_runtime_secs += run_runtime;
accumulate_billing_rollup(
&mut agg,
&fabro_workflow::billing_rollup_from_projection(&final_state),
);
}
let mut runs = state.runs.lock().expect("runs lock poisoned");

317
lib/crates/fabro-server/src/server/handler/billing.rs
View file

@ -1,13 +1,14 @@
use std::collections::HashMap;
use std::sync::Arc;
use fabro_types::EventBody;
use chrono::{DateTime, Utc};
use fabro_types::{RunProjection, StageProjection, StageState};
use super::super::{
ApiError, AppState, BilledTokenCounts, BillingByModel, BillingStageRef, EventEnvelope, HashMap,
IntoResponse, Json, ListResponse, ModelBillingTotals, ModelReference, PaginationParams, Path,
Query, RequiredUser, Response, Router, RunBilling, RunBillingStage, RunBillingTotals, RunId,
RunStage, RunStatus, StageState, State, StatusCode, accumulate_model_billing, get,
parse_run_id_path,
ApiError, AppState, BillingByModel, BillingStageRef, IntoResponse, Json, ListResponse,
ModelReference, PaginationParams, Path, Query, RequiredUser, Response, Router, RunBilling,
RunBillingStage, RunBillingTotals, RunId, State, StatusCode, get, parse_run_id_path,
run_stage_from_stage_id,
};
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -16,25 +17,6 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
.route("/runs/{id}/billing", get(get_run_billing))
}
fn active_stage_state_from_events(events: &[EventEnvelope], node_id: &str) -> StageState {
let latest = events.iter().rev().find(|envelope| {
envelope.event.node_id.as_deref() == Some(node_id)
&& matches!(
&envelope.event.body,
EventBody::StageRetrying(_)
| EventBody::StageStarted(_)
| EventBody::StageCompleted(_)
| EventBody::StageFailed(_)
)
});
if latest.is_some_and(|e| matches!(&e.event.body, EventBody::StageRetrying(_))) {
StageState::Retrying
} else {
StageState::Running
}
}
async fn list_run_stages(
_auth: RequiredUser,
State(state): State<Arc<AppState>>,
@ -46,81 +28,30 @@ async fn list_run_stages(
Err(response) => return response,
};
// Try live run first.
let (checkpoint, run_is_active) = {
let runs = state.runs.lock().expect("runs lock poisoned");
match runs.get(&id) {
Some(managed_run) => {
let active = !matches!(
managed_run.status,
RunStatus::Succeeded { .. } | RunStatus::Failed { .. } | RunStatus::Dead
);
(managed_run.checkpoint.clone(), active)
}
None => (None, false),
let Ok(run_store) = state.store.open_run_reader(&id).await else {
return ApiError::not_found("Run not found.").into_response();
};
let projection = match run_store.state().await {
Ok(state) => state,
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
// Fall back to stored run.
let (checkpoint, run_is_active) = if checkpoint.is_some() {
(checkpoint, run_is_active)
} else {
match state.store.open_run_reader(&id).await {
Ok(run_store) => match run_store.state().await {
Ok(run_state) => {
let active = run_state.status.is_some_and(|status| !status.is_terminal());
(run_state.checkpoint, active)
}
Err(_) => (None, false),
},
Err(_) => return ApiError::not_found("Run not found.").into_response(),
}
};
let Some(checkpoint) = checkpoint else {
return (
StatusCode::OK,
Json(ListResponse::new(Vec::<RunStage>::new())),
)
.into_response();
};
let events = match state.store.open_run_reader(&id).await {
Ok(run_store) => run_store.list_events().await.unwrap_or_default(),
Err(_) => Vec::new(),
};
let stage_durations = fabro_workflow::extract_stage_durations_from_events(&events);
let mut stages = Vec::new();
for node_id in &checkpoint.completed_nodes {
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
let status = match checkpoint.node_outcomes.get(node_id) {
Some(outcome) => StageState::from(outcome.status),
None => StageState::Succeeded,
};
stages.push(RunStage {
id: node_id.clone(),
name: node_id.clone(),
status,
duration_secs: Some(duration_ms as f64 / 1000.0),
dot_id: Some(node_id.clone()),
});
}
// Add next node as running if the run is still active.
// The checkpoint's current_node is the last *completed* stage; next_node_id
// is the stage that is currently executing.
if let Some(next_id) = &checkpoint.next_node_id {
if run_is_active && next_id != "exit" && !checkpoint.completed_nodes.contains(next_id) {
stages.push(RunStage {
id: next_id.clone(),
name: next_id.clone(),
status: active_stage_state_from_events(&events, next_id),
duration_secs: None,
dot_id: Some(next_id.clone()),
});
}
}
let now = Utc::now();
let stages = projection
.iter_stages()
.map(|(stage_id, stage)| {
run_stage_from_stage_id(
stage_id,
stage_id.node_id().to_string(),
stage.effective_state(),
stage.runtime_secs(now),
stage.started_at,
)
})
.collect::<Vec<_>>();
(StatusCode::OK, Json(ListResponse::new(stages))).into_response()
}
@ -137,91 +68,53 @@ async fn get_run_billing(
}
};
let checkpoint = match run_store.state().await {
Ok(state) => state.checkpoint,
let projection = match run_store.state().await {
Ok(state) => state,
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
let Some(checkpoint) = checkpoint else {
let empty = RunBilling {
by_model: Vec::new(),
stages: Vec::new(),
totals: RunBillingTotals {
cache_read_tokens: 0,
cache_write_tokens: 0,
input_tokens: 0,
output_tokens: 0,
reasoning_tokens: 0,
runtime_secs: 0.0,
total_tokens: 0,
total_usd_micros: None,
let rollup = fabro_workflow::billing_rollup_from_projection(&projection);
let by_model = rollup
.by_model
.iter()
.map(|model| BillingByModel {
billing: model.billing.clone(),
model: ModelReference {
id: model.model_id.clone(),
},
};
return (StatusCode::OK, Json(empty)).into_response();
};
stages: model.stages,
})
.collect::<Vec<_>>();
let stage_durations = match run_store.list_events().await {
Ok(events) => fabro_workflow::extract_stage_durations_from_events(&events),
Err(err) => {
return ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string())
.into_response();
}
};
let mut by_model_totals = HashMap::<String, ModelBillingTotals>::new();
let mut billed_usages = Vec::new();
let mut runtime_secs = 0.0_f64;
let mut stages = Vec::new();
for node_id in &checkpoint.completed_nodes {
let duration_ms = stage_durations.get(node_id).copied().unwrap_or(0);
runtime_secs += duration_ms as f64 / 1000.0;
let usage = checkpoint
.node_outcomes
.get(node_id)
.and_then(|outcome| outcome.usage.as_ref());
let (billing, model) = if let Some(usage) = usage {
billed_usages.push(usage.clone());
let tokens = usage.tokens();
let billing = BilledTokenCounts {
cache_read_tokens: tokens.cache_read_tokens,
cache_write_tokens: tokens.cache_write_tokens,
input_tokens: tokens.input_tokens,
output_tokens: tokens.output_tokens,
reasoning_tokens: tokens.reasoning_tokens,
total_tokens: tokens.total_tokens(),
total_usd_micros: usage.total_usd_micros,
};
let model_id = usage.model_id().to_string();
accumulate_model_billing(by_model_totals.entry(model_id.clone()).or_default(), usage);
(billing, Some(ModelReference { id: model_id }))
} else {
(BilledTokenCounts::default(), None)
};
stages.push(RunBillingStage {
billing,
model,
runtime_secs: duration_ms as f64 / 1000.0,
stage: BillingStageRef {
id: node_id.clone(),
name: node_id.clone(),
},
});
}
let totals = BilledTokenCounts::from_billed_usage(&billed_usages);
let by_model = by_model_totals
let rollup_by_node = rollup
.stages
.iter()
.map(|stage| (stage.node_id.as_str(), stage))
.collect::<HashMap<_, _>>();
let live_rows = live_billing_rows(&projection, Utc::now());
let runtime_secs = live_rows.iter().map(|row| row.runtime_secs).sum::<f64>();
let stages = live_rows
.into_iter()
.map(|(model, totals)| BillingByModel {
billing: totals.billing,
model: ModelReference { id: model },
stages: totals.stages,
.map(|row| {
let rollup_stage = rollup_by_node.get(row.node_id.as_str());
RunBillingStage {
billing: rollup_stage
.map(|stage| stage.billing.clone())
.unwrap_or_default(),
model: rollup_stage
.and_then(|stage| stage.model_id.as_ref())
.map(|id| ModelReference { id: id.clone() }),
runtime_secs: row.runtime_secs,
stage: BillingStageRef {
id: row.node_id.clone(),
name: row.node_id,
},
started_at: row.started_at,
state: row.state,
}
})
.collect::<Vec<_>>();
@ -229,16 +122,80 @@ async fn get_run_billing(
by_model,
stages,
totals: RunBillingTotals {
cache_read_tokens: totals.cache_read_tokens,
cache_write_tokens: totals.cache_write_tokens,
input_tokens: totals.input_tokens,
output_tokens: totals.output_tokens,
reasoning_tokens: totals.reasoning_tokens,
cache_read_tokens: rollup.totals.cache_read_tokens,
cache_write_tokens: rollup.totals.cache_write_tokens,
input_tokens: rollup.totals.input_tokens,
output_tokens: rollup.totals.output_tokens,
reasoning_tokens: rollup.totals.reasoning_tokens,
runtime_secs,
total_tokens: totals.total_tokens,
total_usd_micros: totals.total_usd_micros,
total_tokens: rollup.totals.total_tokens,
total_usd_micros: rollup.totals.total_usd_micros,
},
};
(StatusCode::OK, Json(response)).into_response()
}
struct LiveBillingRow {
node_id: String,
runtime_secs: f64,
started_at: Option<DateTime<Utc>>,
state: Option<StageState>,
latest_visit: u32,
}
fn live_billing_rows(projection: &RunProjection, now: DateTime<Utc>) -> Vec<LiveBillingRow> {
let mut row_indices = HashMap::<String, usize>::new();
let mut rows = Vec::<LiveBillingRow>::new();
for (stage_id, stage) in projection.iter_stages() {
let node_id = stage_id.node_id();
if is_exit_stage(projection, node_id) || !stage_has_billing_row(stage) {
continue;
}
let index = *row_indices.entry(node_id.to_string()).or_insert_with(|| {
let index = rows.len();
rows.push(LiveBillingRow {
node_id: node_id.to_string(),
runtime_secs: 0.0,
started_at: None,
state: None,
latest_visit: 0,
});
index
});
let row = &mut rows[index];
row.runtime_secs += billing_runtime_secs(stage, now).unwrap_or(0.0);
if stage_id.visit() >= row.latest_visit {
row.latest_visit = stage_id.visit();
row.started_at = stage.started_at;
row.state = Some(stage.effective_state());
}
}
rows
}
fn billing_runtime_secs(stage: &StageProjection, now: DateTime<Utc>) -> Option<f64> {
stage
.duration_ms
.map(|ms| ms as f64 / 1000.0)
.or_else(|| stage.runtime_secs(now))
}
fn stage_has_billing_row(stage: &StageProjection) -> bool {
stage.completion.is_some()
|| stage.duration_ms.is_some()
|| stage.usage.is_some()
|| stage.started_at.is_some()
|| stage.state.is_some()
}
fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool {
projection
.spec()
.and_then(|spec| spec.graph().nodes.get(node_id))
.is_some_and(|node| node.handler_type() == Some("exit"))
}

311
lib/crates/fabro-server/src/server/handler/events.rs
View file

@ -3,9 +3,10 @@ use std::sync::Arc;
use super::super::{
ApiError, AppState, AppendEventResponse, BroadcastStream, Event, EventBody, EventEnvelope,
EventPayload, HashSet, IntoResponse, Json, KeepAlive, PaginatedEventList, PaginationMeta, Path,
Query, RequireRunScoped, RequiredUser, Response, Router, RunEvent, RunId, RunStatus, Sse,
State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get, mpsc, parse_run_id_path,
redact_jsonl_line, reject_if_archived, update_live_run_from_event,
Query, RequireRunScoped, RequireRunStageScoped, RequiredUser, Response, Router, RunEvent,
RunId, RunStatus, Sse, State, StatusCode, StreamExt, UnboundedReceiverStream, broadcast, get,
mpsc, parse_run_id_path, parse_stage_id_path, redact_jsonl_line, reject_if_archived,
update_live_run_from_event,
};
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -15,11 +16,15 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
"/runs/{id}/events",
get(list_run_events).post(append_run_event),
)
.route(
"/runs/{id}/stages/{stageId}/events",
get(list_run_stage_events),
)
.route("/runs/{id}/attach", get(attach_run_events))
}
#[derive(serde::Deserialize)]
struct EventListParams {
pub(crate) struct EventListParams {
#[serde(default)]
since_seq: Option<u32>,
#[serde(default)]
@ -27,11 +32,11 @@ struct EventListParams {
}
impl EventListParams {
fn since_seq(&self) -> u32 {
pub(crate) fn since_seq(&self) -> u32 {
self.since_seq.unwrap_or(1).max(1)
}
fn limit(&self) -> usize {
pub(crate) fn limit(&self) -> usize {
self.limit.unwrap_or(100).clamp(1, 1000)
}
}
@ -200,6 +205,39 @@ async fn list_run_events(
}
}
async fn list_run_stage_events(
RequireRunStageScoped(id, stage_id): RequireRunStageScoped,
State(state): State<Arc<AppState>>,
Query(params): Query<EventListParams>,
) -> Response {
let stage_id = match parse_stage_id_path(&stage_id) {
Ok(stage_id) => stage_id,
Err(response) => return response,
};
let since_seq = params.since_seq();
let limit = params.limit();
match state.store.open_run_reader(&id).await {
Ok(run_store) => match run_store
.list_events_for_stage_from_with_limit(&stage_id, since_seq, limit)
.await
{
Ok(mut events) => {
let has_more = events.len() > limit;
events.truncate(limit);
Json(PaginatedEventList {
data: events,
meta: PaginationMeta { has_more },
})
.into_response()
}
Err(err) => {
ApiError::new(StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response()
}
},
Err(_) => ApiError::not_found("Run not found.").into_response(),
}
}
async fn attach_run_events(
_auth: RequiredUser,
State(state): State<Arc<AppState>>,
@ -342,3 +380,264 @@ fn denied_lifecycle_event_name(body: &EventBody) -> Option<&'static str> {
_ => None,
}
}
#[cfg(test)]
mod stage_events_tests {
use axum::body::{Body, to_bytes};
use axum::http::{Request, StatusCode, header};
use fabro_store::EventPayload;
use fabro_types::RunId;
use serde_json::json;
use tower::ServiceExt;
use crate::test_support::{build_test_router, test_app_state};
fn req_get(uri: &str) -> Request<Body> {
Request::builder()
.method("GET")
.uri(uri)
.body(Body::empty())
.expect("stage events GET request should build")
}
fn make_event(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload {
make_event_with_stage_id(run_id, idx, node_id, None)
}
fn make_event_with_stage_id(
run_id: &RunId,
idx: u32,
node_id: Option<&str>,
stage_id: Option<&str>,
) -> EventPayload {
let mut value = json!({
"id": format!("evt-{idx}"),
"ts": "2026-04-09T12:00:00Z",
"run_id": run_id.to_string(),
"event": "stage.prompt",
"properties": {
"visit": 1,
"text": format!("prompt {idx}"),
},
});
if let Some(node) = node_id {
value
.as_object_mut()
.unwrap()
.insert("node_id".into(), json!(node));
}
if let Some(stage_id) = stage_id {
value
.as_object_mut()
.unwrap()
.insert("stage_id".into(), json!(stage_id));
}
EventPayload::new(value, run_id).expect("event payload should validate")
}
async fn body_json(response: axum::response::Response) -> serde_json::Value {
let bytes = to_bytes(response.into_body(), usize::MAX)
.await
.expect("response body should fit in memory");
serde_json::from_slice(&bytes).expect("response body should be valid JSON")
}
async fn seed_run_with_mixed_events() -> (RunId, axum::Router) {
let state = test_app_state();
let app = build_test_router(state.clone());
let run_id = RunId::new();
let run_store = state
.store_ref()
.create_run(&run_id)
.await
.expect("test run should be creatable");
// Seed 200 unrelated 'beta' events first so any node-blind
// truncation would lose the sparse 'alpha' tail. Then 3 'alpha'
// events past seq 100, plus a couple with no node_id at all.
for idx in 1..=200_u32 {
run_store
.append_event(&make_event(&run_id, idx, Some("beta")))
.await
.expect("append should succeed");
}
run_store
.append_event(&make_event(&run_id, 201, None))
.await
.expect("append should succeed");
for idx in 202..=204_u32 {
run_store
.append_event(&make_event(&run_id, idx, Some("alpha")))
.await
.expect("append should succeed");
}
(run_id, app)
}
#[tokio::test]
async fn returns_only_matching_node_events_in_seq_order() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha@1/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let data = body["data"].as_array().expect("data is array");
let seqs: Vec<u64> = data.iter().map(|e| e["seq"].as_u64().unwrap()).collect();
assert_eq!(seqs, vec![202, 203, 204]);
assert_eq!(body["meta"]["has_more"], false);
}
#[tokio::test]
async fn since_seq_filters_to_events_with_seq_at_least_k() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha@1/events?since_seq=203"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let seqs: Vec<u64> = body["data"]
.as_array()
.unwrap()
.iter()
.map(|e| e["seq"].as_u64().unwrap())
.collect();
assert_eq!(seqs, vec![203, 204]);
}
#[tokio::test]
async fn limit_one_returns_first_envelope_with_has_more_true() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/alpha@1/events?limit=1"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let data = body["data"].as_array().unwrap();
assert_eq!(data.len(), 1);
assert_eq!(data[0]["seq"].as_u64().unwrap(), 202);
assert_eq!(body["meta"]["has_more"], true);
}
#[tokio::test]
async fn unknown_stage_in_existing_run_returns_empty_list_with_no_more() {
let (run_id, app) = seed_run_with_mixed_events().await;
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/unknown-stage@1/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
assert_eq!(body["data"].as_array().unwrap().len(), 0);
assert_eq!(body["meta"]["has_more"], false);
}
#[tokio::test]
async fn missing_run_returns_404_with_run_not_found() {
let app = build_test_router(test_app_state());
// A syntactically valid RunId that the store has never seen, so
// `parse_run_id_path` succeeds but `open_run_reader` fails — that
// exercises the handler's not-found branch rather than the path
// parser's 400 branch.
let absent = RunId::new();
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{absent}/stages/alpha@1/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::NOT_FOUND);
let body = body_json(response).await;
let detail = body["errors"][0]["detail"]
.as_str()
.expect("error detail string");
assert!(
detail.contains("Run not found."),
"unexpected error body: {body}"
);
}
#[tokio::test]
async fn unauthenticated_request_is_rejected() {
let state = test_app_state();
// Bypass `build_test_router`'s auto-injected bearer token by
// building the raw router directly. The principal middleware sees
// a missing Authorization header and the extractor enforces auth.
let app = crate::server::build_router(state, crate::test_support::test_auth_mode());
let run_id = RunId::new();
let request = Request::builder()
.method("GET")
.uri(format!("/api/v1/runs/{run_id}/stages/alpha@1/events"))
.header(header::ACCEPT, "application/json")
.body(Body::empty())
.unwrap();
let response = app.oneshot(request).await.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn returns_only_requested_visit_when_stage_id_is_present() {
let state = test_app_state();
let app = build_test_router(state.clone());
let run_id = RunId::new();
let run_store = state
.store_ref()
.create_run(&run_id)
.await
.expect("test run should be creatable");
run_store
.append_event(&make_event_with_stage_id(
&run_id,
1,
Some("verify"),
Some("verify@1"),
))
.await
.expect("append should succeed");
run_store
.append_event(&make_event_with_stage_id(
&run_id,
2,
Some("verify"),
Some("verify@2"),
))
.await
.expect("append should succeed");
let response = app
.oneshot(req_get(&format!(
"/api/v1/runs/{run_id}/stages/verify@2/events"
)))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
let body = body_json(response).await;
let seqs: Vec<u64> = body["data"]
.as_array()
.unwrap()
.iter()
.map(|e| e["seq"].as_u64().unwrap())
.collect();
assert_eq!(seqs, vec![2]);
}
}

16
lib/crates/fabro-server/src/server/handler/lifecycle.rs
View file

@ -1,13 +1,13 @@
use std::sync::Arc;
use super::super::{
ApiError, AppState, FailureReason, ForkRequest, ForkResponse, IntoResponse, Json, Ordering,
Path, Principal, RequiredUser, Response, RewindRequest, RewindResponse, Router,
RunAnswerTransport, RunControlAction, RunExecutionMode, RunId, RunStatus, RunStatusResponse,
StartRunRequest, State, StatusCode, Storage, TimelineEntryResponse, WORKER_CANCEL_GRACE,
WorkflowError, append_control_request, get, load_pending_control, managed_run, operations,
parse_run_id_path, persist_cancelled_run_status, post, reject_if_archived, sleep,
update_live_run_from_event, workflow_event,
ApiError, AppState, FailureReason, ForkRequest, ForkResponse, IntoResponse, Json, Path,
Principal, RequiredUser, Response, RewindRequest, RewindResponse, Router, RunAnswerTransport,
RunControlAction, RunExecutionMode, RunId, RunStatus, RunStatusResponse, StartRunRequest,
State, StatusCode, Storage, TimelineEntryResponse, WORKER_CANCEL_GRACE, WorkflowError,
append_control_request, get, load_pending_control, managed_run, operations, parse_run_id_path,
persist_cancelled_run_status, post, reject_if_archived, sleep, update_live_run_from_event,
workflow_event,
};
pub(super) fn routes() -> Router<Arc<AppState>> {
@ -249,7 +249,7 @@ async fn cancel_run(
}
if let Some(token) = &cancel_token {
token.store(true, Ordering::SeqCst);
token.cancel();
}
let sent_cancel_signal = if let Some(cancel_tx) = cancel_tx {
let _ = cancel_tx.send(());

4
lib/crates/fabro-server/src/server/handler/mod.rs
View file

@ -58,8 +58,8 @@ pub(super) fn demo_routes() -> Router<Arc<AppState>> {
.route("/runs/{id}/artifacts", get(demo::list_run_artifacts_stub))
.route("/runs/{id}/files", get(demo::list_run_files_stub))
.route(
"/runs/{id}/stages/{stageId}/turns",
get(demo::get_stage_turns),
"/runs/{id}/stages/{stageId}/events",
get(demo::get_stage_events),
)
.route(
"/runs/{id}/stages/{stageId}/artifacts",

52
lib/crates/fabro-server/src/server/handler/runs.rs
View file

@ -10,7 +10,9 @@ use axum::{Json, Router};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use bytes::Bytes;
use fabro_api::types::{RunManifest, RunStatusResponse, SubmitAnswerRequest};
use fabro_api::types::{
BoardColumn, BoardColumnDefinition, RunManifest, RunStatusResponse, SubmitAnswerRequest,
};
use fabro_config::Storage;
use fabro_interview::AnswerSubmission;
use fabro_types::{
@ -82,25 +84,45 @@ impl ListRunsParams {
}
}
fn board_column(status: RunStatus) -> Option<&'static str> {
fn board_column(status: RunStatus) -> Option<BoardColumn> {
match status {
RunStatus::Submitted | RunStatus::Queued | RunStatus::Starting => Some("initializing"),
RunStatus::Running | RunStatus::Paused { .. } => Some("running"),
RunStatus::Blocked { .. } => Some("blocked"),
RunStatus::Succeeded { .. } => Some("succeeded"),
RunStatus::Failed { .. } | RunStatus::Dead => Some("failed"),
RunStatus::Submitted | RunStatus::Queued => Some(BoardColumn::Queued),
RunStatus::Starting => Some(BoardColumn::Initializing),
RunStatus::Running | RunStatus::Paused { .. } => Some(BoardColumn::Running),
RunStatus::Blocked { .. } => Some(BoardColumn::Blocked),
RunStatus::Succeeded { .. } => Some(BoardColumn::Succeeded),
RunStatus::Failed { .. } | RunStatus::Dead => Some(BoardColumn::Failed),
RunStatus::Removing | RunStatus::Archived { .. } => None,
}
}
pub(crate) fn board_columns() -> serde_json::Value {
serde_json::json!([
{"id": "initializing", "name": "Initializing"},
{"id": "running", "name": "Running"},
{"id": "blocked", "name": "Blocked"},
{"id": "succeeded", "name": "Succeeded"},
{"id": "failed", "name": "Failed"},
])
pub(crate) fn board_columns() -> Vec<BoardColumnDefinition> {
vec![
BoardColumnDefinition {
id: BoardColumn::Queued,
name: "Queued".into(),
},
BoardColumnDefinition {
id: BoardColumn::Initializing,
name: "Initializing".into(),
},
BoardColumnDefinition {
id: BoardColumn::Running,
name: "Running".into(),
},
BoardColumnDefinition {
id: BoardColumn::Blocked,
name: "Blocked".into(),
},
BoardColumnDefinition {
id: BoardColumn::Succeeded,
name: "Succeeded".into(),
},
BoardColumnDefinition {
id: BoardColumn::Failed,
name: "Failed".into(),
},
]
}
async fn board_run_metadata(

999
lib/crates/fabro-server/src/server/tests.rs
View file

File diff suppressed because it is too large Load diff

75
lib/crates/fabro-server/tests/it/event_pagination.rs Normal file
View file

@ -0,0 +1,75 @@
//! Cursor-pagination tests for the per-stage events endpoint (demo mode).
//!
//! The stage-events route uses `since_seq=` + `limit=` (cursor-based) instead
//! of the offset-based `page[limit]/page[offset]` pagination used by other
//! list endpoints, so it gets its own test rather than living in the generic
//! offset-shape matrix.
#![allow(
clippy::absolute_paths,
reason = "This test module prefers explicit type paths over extra imports."
)]
use axum::body::Body;
use axum::http::{Request, StatusCode};
use tower::ServiceExt;
use super::helpers::{response_json, test_app_state};
async fn get_json(app: &axum::Router, uri: &str) -> serde_json::Value {
let req = Request::builder()
.method("GET")
.uri(uri)
.header("x-fabro-demo", "1")
.body(Body::empty())
.expect("event pagination request should build");
let response = app.clone().oneshot(req).await.unwrap();
response_json(response, StatusCode::OK, format!("GET {uri}")).await
}
#[tokio::test]
async fn demo_stage_events_default_returns_all_fixture_events_with_no_more() {
let app = fabro_server::test_support::build_test_router(test_app_state());
let body = get_json(&app, "/api/v1/runs/run-1/stages/detect-drift@1/events").await;
let data = body["data"].as_array().expect("data is an array");
assert_eq!(data.len(), 7, "all seven fixture events should be returned");
assert_eq!(body["meta"]["has_more"], false);
}
#[tokio::test]
async fn demo_stage_events_limit_one_signals_has_more() {
let app = fabro_server::test_support::build_test_router(test_app_state());
let body = get_json(
&app,
"/api/v1/runs/run-1/stages/detect-drift@1/events?limit=1",
)
.await;
let data = body["data"].as_array().expect("data is an array");
assert_eq!(data.len(), 1);
assert_eq!(body["meta"]["has_more"], true);
}
#[tokio::test]
async fn demo_stage_events_since_seq_filters_out_earlier_events() {
let app = fabro_server::test_support::build_test_router(test_app_state());
// The fixture seqs are 1..=7. since_seq=4 should skip the first three.
let body = get_json(
&app,
"/api/v1/runs/run-1/stages/detect-drift@1/events?since_seq=4",
)
.await;
let data = body["data"].as_array().expect("data is an array");
assert_eq!(data.len(), 4);
let seqs: Vec<u64> = data
.iter()
.map(|envelope| envelope["seq"].as_u64().expect("seq is a number"))
.collect();
assert_eq!(seqs, vec![4, 5, 6, 7]);
assert_eq!(body["meta"]["has_more"], false);
}

1
lib/crates/fabro-server/tests/it/main.rs
View file

@ -4,6 +4,7 @@
)]
mod api;
mod event_pagination;
mod helpers;
mod openapi_conformance;
mod pagination;

4
lib/crates/fabro-server/tests/it/pagination.rs
View file

@ -56,10 +56,6 @@ const ENDPOINTS: &[PaginatedEndpoint] = &[
path: "/api/v1/models",
name: "listModels",
},
PaginatedEndpoint {
path: "/api/v1/runs/run-1/stages/detect-drift/turns",
name: "listStageTurns",
},
PaginatedEndpoint {
path: "/api/v1/runs/run-1/questions",
name: "listRunQuestions",

7
lib/crates/fabro-store/src/artifact_store.rs
View file

@ -297,8 +297,13 @@ fn decode_artifact_location(
))
})?;
let (retry, filename) = decode_retry_and_filename(location, &mut parts)?;
let stage_id = StageId::try_new(node_id, visit).map_err(|err| {
Error::Other(format!(
"artifact location {location} has an invalid stage id: {err}"
))
})?;
Ok(NodeArtifact {
node: StageId::new(node_id, visit),
node: stage_id,
retry,
filename,
size,

722
lib/crates/fabro-store/src/run_state.rs
View file

@ -7,10 +7,11 @@ use fabro_types::run_event::{
RunFailedProps, StageCompletedProps, StagePromptProps,
};
use fabro_types::{
BilledModelUsage, Checkpoint, Conclusion, EventBody, FailureSignature, InterviewQuestionRecord,
Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction, RunEvent, RunId,
RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion, StageOutcome,
StageProjection, StartRecord, TerminalStatus, first_event_seq,
BilledModelUsage, Checkpoint, CommandTermination, Conclusion, EventBody, FailureSignature,
InterviewQuestionRecord, Outcome, PendingInterviewRecord, PullRequestRecord, RunControlAction,
RunEvent, RunId, RunProjection, RunSpec, RunStatus, RunSummary, SandboxRecord, StageCompletion,
StageId, StageOutcome, StageProjection, StageState, StartRecord, TerminalStatus,
first_event_seq,
};
use fabro_util::error::render_with_causes;
use serde_json::Value;
@ -290,59 +291,76 @@ impl RunProjectionReducer for RunProjection {
let Some(stage_id) = stored.stage_id.as_ref() else {
return Ok(());
};
self.stage_entry(
let stage = self.stage_entry(
stage_id.node_id(),
stage_id.visit(),
first_event_seq(event.seq),
);
stage.begin_attempt(ts);
}
EventBody::StageRetrying(_) => {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.state = Some(StageState::Retrying);
}
EventBody::StagePrompt(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.prompt = Some(props.text.clone());
stage.provider_used = provider_used_from_prompt(props);
}
EventBody::PromptCompleted(props) => {
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.response = Some(props.response.clone());
}
EventBody::StageCompleted(props) => {
let Some(node_id) = stored.node_id.as_deref() else {
return Ok(());
};
let visit = stage_visit(node_id, props.node_visits.as_ref(), self).unwrap_or(1);
let response = props.response.clone();
let outcome = stage_outcome_from_props(props);
let completion = stage_completion_from_outcome(&outcome, ts);
let stage = self.stage_entry(node_id, visit, first_event_seq(event.seq));
let Some(stage) =
stage_at_completed_visit(self, stored, props.node_visits.as_ref(), event.seq)
else {
return Ok(());
};
stage.response = response;
stage.completion = Some(completion);
stage.duration_ms = Some(props.duration_ms);
stage.usage.clone_from(&props.billing);
stage.state = Some(StageState::from(outcome.status));
}
EventBody::StageFailed(props) => {
let failure_reason = props.failure.as_ref().map(|detail| detail.message.clone());
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
let outcome = StageOutcome::Failed {
retry_requested: props.will_retry,
};
stage.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: false,
},
outcome,
notes: None,
failure_reason,
timestamp: ts,
});
stage.duration_ms = Some(props.duration_ms);
stage.usage.clone_from(&props.billing);
stage.state = Some(StageState::from(outcome));
}
EventBody::AgentSessionStarted(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.provider_used = Some(provider_used_from_agent_session_started(props));
}
EventBody::AgentCliStarted(props) => {
let Some(stage) = stage_at_visit(self, stored, props.visit, event.seq) else {
let Some(stage) = stage_at_stored_or_visit(self, stored, props.visit, event.seq)
else {
return Ok(());
};
stage.provider_used = Some(provider_used_from_agent_cli_started(props));
@ -351,7 +369,7 @@ impl RunProjectionReducer for RunProjection {
let script_invocation = serde_json::to_value(props).map_err(|err| {
Error::InvalidEvent(format!("invalid command.started payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.script_invocation = Some(script_invocation);
@ -360,7 +378,7 @@ impl RunProjectionReducer for RunProjection {
let script_timing = serde_json::to_value(props).map_err(|err| {
Error::InvalidEvent(format!("invalid command.completed payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.stdout = Some(props.stdout.clone());
@ -372,11 +390,47 @@ impl RunProjectionReducer for RunProjection {
stage.termination = Some(props.termination);
stage.script_timing = Some(script_timing);
}
EventBody::AgentCliCompleted(props) => {
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
return Ok(());
};
apply_agent_cli_terminal(
stage,
props,
&props.stdout,
&props.stderr,
CommandTermination::Exited,
)?;
}
EventBody::AgentCliCancelled(props) => {
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
return Ok(());
};
apply_agent_cli_terminal(
stage,
props,
&props.stdout,
&props.stderr,
CommandTermination::Cancelled,
)?;
}
EventBody::AgentCliTimedOut(props) => {
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
return Ok(());
};
apply_agent_cli_terminal(
stage,
props,
&props.stdout,
&props.stderr,
CommandTermination::TimedOut,
)?;
}
EventBody::ParallelCompleted(props) => {
let parallel_results = serde_json::to_value(&props.results).map_err(|err| {
Error::InvalidEvent(format!("invalid parallel.completed payload: {err}"))
})?;
let Some(stage) = stage_at_current_visit(self, stored, event.seq) else {
let Some(stage) = stage_at_stored_or_current_visit(self, stored, event.seq) else {
return Ok(());
};
stage.parallel_results = Some(parallel_results);
@ -394,6 +448,9 @@ fn stage_at_visit<'a>(
visit: u32,
seq: u32,
) -> Option<&'a mut StageProjection> {
if visit == 0 {
return None;
}
let node_id = stored.node_id.as_deref()?;
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
@ -408,6 +465,51 @@ fn stage_at_current_visit<'a>(
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
fn stage_at_stored_stage_id<'a>(
state: &'a mut RunProjection,
stage_id: &StageId,
seq: u32,
) -> &'a mut StageProjection {
state.stage_entry(stage_id.node_id(), stage_id.visit(), first_event_seq(seq))
}
fn stage_at_stored_or_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
visit: u32,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
stage_at_visit(state, stored, visit, seq)
}
fn stage_at_stored_or_current_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
stage_at_current_visit(state, stored, seq)
}
fn stage_at_completed_visit<'a>(
state: &'a mut RunProjection,
stored: &RunEvent,
node_visits: Option<&BTreeMap<String, usize>>,
seq: u32,
) -> Option<&'a mut StageProjection> {
if let Some(stage_id) = stored.stage_id.as_ref() {
return Some(stage_at_stored_stage_id(state, stage_id, seq));
}
let node_id = stored.node_id.as_deref()?;
let visit = stage_visit(node_id, node_visits, state).unwrap_or(1);
Some(state.stage_entry(node_id, visit, first_event_seq(seq)))
}
pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> RunSummary {
let workflow_name = state.spec.as_ref().map(|spec| {
if spec.graph.name.is_empty() {
@ -529,6 +631,7 @@ fn stage_visit(
node_visits
.and_then(|visits| visits.get(node_id).copied())
.and_then(|visit| u32::try_from(visit).ok())
.filter(|visit| *visit > 0)
.or_else(|| state.current_visit_for(node_id))
}
@ -605,6 +708,22 @@ fn provider_used_from_agent_cli_started(props: &AgentCliStartedProps) -> Value {
Value::Object(provider_used)
}
fn apply_agent_cli_terminal(
stage: &mut StageProjection,
props: &impl serde::Serialize,
stdout: &str,
stderr: &str,
termination: CommandTermination,
) -> Result<()> {
let script_timing = serde_json::to_value(props)
.map_err(|err| Error::InvalidEvent(format!("invalid agent.cli terminal payload: {err}")))?;
stage.stdout = Some(stdout.to_string());
stage.stderr = Some(stderr.to_string());
stage.termination = Some(termination);
stage.script_timing = Some(script_timing);
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::{BTreeMap, HashMap};
@ -612,13 +731,16 @@ mod tests {
use chrono::Utc;
use fabro_types::run_event::run::RunFailedProps;
use fabro_types::run_event::{
AgentCliCancelledProps, AgentCliCompletedProps, AgentCliTimedOutProps,
CheckpointCompletedProps, InterviewCompletedProps, InterviewOption, InterviewStartedProps,
RunControlEffectProps, StagePromptProps, StageStartedProps,
RunControlEffectProps, StageCompletedProps, StageFailedProps, StagePromptProps,
StageRetryingProps, StageStartedProps,
};
use fabro_types::{
BlockedReason, Checkpoint, EventBody, FailureReason, Outcome, QuestionType, RunBlobId,
RunControlAction, RunEvent, RunStatus, StageOutcome, SuccessReason, TerminalStatus,
WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, BlockedReason, Checkpoint, CommandTermination, EventBody,
FailureCategory, FailureDetail, FailureReason, Outcome, QuestionType, RunBlobId,
RunControlAction, RunEvent, RunStatus, StageOutcome, StageState, SuccessReason,
TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
@ -651,6 +773,32 @@ mod tests {
event
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
fn usage_json(usage: &BilledModelUsage) -> serde_json::Value {
serde_json::to_value(usage).unwrap()
}
fn test_raw_event(
seq: u32,
event: &str,
@ -861,6 +1009,310 @@ mod tests {
assert_eq!(stage.prompt.as_deref(), Some("prompt"));
}
fn start_stage(state: &mut RunProjection, stage_id: &StageId) {
state
.apply_event(&test_stage_event(
3,
EventBody::StageStarted(StageStartedProps {
index: 0,
handler_type: "agent".to_string(),
attempt: 1,
max_attempts: 1,
}),
stage_id.clone(),
))
.unwrap();
}
#[test]
fn agent_cli_completed_updates_stage_output_projection() {
let mut state = RunProjection::default();
let stage_id = StageId::new("code", 1);
start_stage(&mut state, &stage_id);
state
.apply_event(&test_stage_event(
4,
EventBody::AgentCliCompleted(AgentCliCompletedProps {
stdout: "done".to_string(),
stderr: "warn".to_string(),
exit_code: 0,
duration_ms: 42,
}),
stage_id.clone(),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.stdout.as_deref(), Some("done"));
assert_eq!(stage.stderr.as_deref(), Some("warn"));
assert_eq!(stage.termination, Some(CommandTermination::Exited));
assert_eq!(
stage.script_timing.as_ref().unwrap()["duration_ms"],
serde_json::json!(42)
);
}
#[test]
fn agent_cli_cancelled_updates_stage_output_projection() {
let mut state = RunProjection::default();
let stage_id = StageId::new("code", 1);
start_stage(&mut state, &stage_id);
state
.apply_event(&test_stage_event(
4,
EventBody::AgentCliCancelled(AgentCliCancelledProps {
stdout: "partial".to_string(),
stderr: "cancelled".to_string(),
duration_ms: 7,
}),
stage_id.clone(),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.stdout.as_deref(), Some("partial"));
assert_eq!(stage.stderr.as_deref(), Some("cancelled"));
assert_eq!(stage.termination, Some(CommandTermination::Cancelled));
assert_eq!(
stage.script_timing.as_ref().unwrap()["duration_ms"],
serde_json::json!(7)
);
}
#[test]
fn agent_cli_timed_out_updates_stage_output_projection() {
let mut state = RunProjection::default();
let stage_id = StageId::new("code", 1);
start_stage(&mut state, &stage_id);
state
.apply_event(&test_stage_event(
4,
EventBody::AgentCliTimedOut(AgentCliTimedOutProps {
stdout: "partial".to_string(),
stderr: "timeout".to_string(),
duration_ms: 600,
}),
stage_id.clone(),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.stdout.as_deref(), Some("partial"));
assert_eq!(stage.stderr.as_deref(), Some("timeout"));
assert_eq!(stage.termination, Some(CommandTermination::TimedOut));
assert_eq!(
stage.script_timing.as_ref().unwrap()["duration_ms"],
serde_json::json!(600)
);
}
#[test]
fn stage_completed_event_captures_duration_and_usage_per_visit() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 123, 45);
state
.apply_event(&test_event(
3,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms: 789,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("done".to_string()),
attempt: 1,
max_attempts: 1,
}),
Some("build"),
))
.unwrap();
let stage = state.stage(&StageId::new("build", 1)).unwrap();
assert_eq!(stage.duration_ms, Some(789));
assert_eq!(stage.usage.as_ref(), Some(&usage));
}
#[test]
fn stage_failed_event_captures_duration_and_usage_per_visit() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
let usage = test_usage("gpt-5.2", 321, 54);
state
.apply_event(&test_stage_event(
2,
EventBody::StageStarted(StageStartedProps {
index: 0,
handler_type: "agent".to_string(),
attempt: 1,
max_attempts: 1,
}),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_raw_event(
3,
"stage.failed",
&json!({
"index": 0,
"failure": {
"message": "provider failed",
"failure_class": "transient_infra"
},
"will_retry": false,
"duration_ms": 654,
"billing": usage_json(&usage)
}),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(654));
assert_eq!(stage.usage.as_ref(), Some(&usage));
}
#[test]
fn two_visits_of_one_node_retain_distinct_usage() {
let mut state = RunProjection::default();
let first_usage = test_usage("gpt-5.2", 100, 10);
let second_usage = test_usage("gpt-5.2", 200, 20);
for (seq, visit, duration_ms, usage) in [
(3, 1usize, 111, first_usage.clone()),
(4, 2usize, 222, second_usage.clone()),
] {
state
.apply_event(&test_event(
seq,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: Some(BTreeMap::from([("build".to_string(), visit)])),
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 1,
}),
Some("build"),
))
.unwrap();
}
let first_stage = state.stage(&StageId::new("build", 1)).unwrap();
let second_stage = state.stage(&StageId::new("build", 2)).unwrap();
assert_eq!(first_stage.duration_ms, Some(111));
assert_eq!(first_stage.usage.as_ref(), Some(&first_usage));
assert_eq!(second_stage.duration_ms, Some(222));
assert_eq!(second_stage.usage.as_ref(), Some(&second_usage));
}
#[test]
fn stage_completed_prefers_stored_stage_id_over_legacy_node_visits() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 300, 30);
let scoped_stage_id = StageId::new("build", 2);
state
.apply_event(&test_stage_event(
3,
EventBody::StageCompleted(StageCompletedProps {
index: 0,
duration_ms: 333,
status: StageOutcome::Succeeded,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: Some(usage.clone()),
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: Some(BTreeMap::from([("build".to_string(), 1usize)])),
loop_failure_signatures: None,
restart_failure_signatures: None,
response: Some("done".to_string()),
attempt: 1,
max_attempts: 1,
}),
scoped_stage_id.clone(),
))
.unwrap();
assert!(
state.stage(&StageId::new("build", 1)).is_none(),
"legacy node_visits must not override stored stage_id"
);
let stage = state.stage(&scoped_stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(333));
assert_eq!(stage.usage.as_ref(), Some(&usage));
assert_eq!(stage.response.as_deref(), Some("done"));
}
#[test]
fn stage_failed_prefers_stored_stage_id_and_preserves_retry_request() {
let mut state = RunProjection::default();
let usage = test_usage("gpt-5.2", 400, 40);
let scoped_stage_id = StageId::new("build", 2);
state
.apply_event(&test_stage_event(
3,
EventBody::StageFailed(StageFailedProps {
index: 0,
failure: Some(fabro_types::FailureDetail::new(
"try again",
fabro_types::FailureCategory::TransientInfra,
)),
will_retry: true,
duration_ms: 444,
billing: Some(usage.clone()),
}),
scoped_stage_id.clone(),
))
.unwrap();
assert!(
state.stage(&StageId::new("build", 1)).is_none(),
"current-visit fallback must not override stored stage_id"
);
let stage = state.stage(&scoped_stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(444));
assert_eq!(stage.usage.as_ref(), Some(&usage));
let completion = stage.completion.as_ref().unwrap();
assert_eq!(completion.outcome, StageOutcome::Failed {
retry_requested: true,
});
assert_eq!(completion.failure_reason.as_deref(), Some("try again"));
}
#[test]
fn checkpoint_completed_creates_projection_entry_for_skipped_stage() {
let mut state = RunProjection::default();
@ -1488,4 +1940,224 @@ mod tests {
);
assert_eq!(state.status_updated_at, updated_at);
}
fn started_props() -> StageStartedProps {
StageStartedProps {
index: 0,
handler_type: "agent".to_string(),
attempt: 1,
max_attempts: 3,
}
}
fn failed_props(duration_ms: u64, will_retry: bool) -> StageFailedProps {
StageFailedProps {
index: 0,
failure: Some(FailureDetail::new("boom", FailureCategory::TransientInfra)),
will_retry,
duration_ms,
billing: None,
}
}
fn retrying_props() -> StageRetryingProps {
StageRetryingProps {
index: 0,
attempt: 2,
max_attempts: 3,
delay_ms: 0,
}
}
fn completed_props(duration_ms: u64, status: StageOutcome) -> StageCompletedProps {
StageCompletedProps {
index: 0,
duration_ms,
status,
preferred_label: None,
suggested_next_ids: Vec::new(),
billing: None,
failure: None,
notes: None,
files_touched: Vec::new(),
context_updates: None,
jump_to_node: None,
context_values: None,
node_visits: None,
loop_failure_signatures: None,
restart_failure_signatures: None,
response: None,
attempt: 1,
max_attempts: 3,
}
}
fn billed_usage() -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": "gpt-test"
},
"tokens": {
"input_tokens": 10,
"output_tokens": 5,
"reasoning_tokens": 2,
"cache_read_tokens": 3,
"cache_write_tokens": 4
}
},
"facts": { "provider": "open_ai" }
},
"total_usd_micros": 123
}))
.expect("billing fixture should deserialize")
}
#[test]
fn stage_started_records_started_at_and_running_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
3,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.state, Some(StageState::Running));
assert!(stage.started_at.is_some());
assert_eq!(stage.effective_state(), StageState::Running);
}
#[test]
fn stage_completed_records_duration_usage_and_terminal_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
let usage = billed_usage();
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
let mut props = completed_props(42, StageOutcome::Succeeded);
props.billing = Some(usage.clone());
state
.apply_event(&test_event(
2,
EventBody::StageCompleted(props),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(42));
assert_eq!(stage.usage.as_ref(), Some(&usage));
assert_eq!(stage.state, Some(StageState::Succeeded));
assert_eq!(stage.effective_state(), StageState::Succeeded);
}
#[test]
fn stage_failed_records_duration_and_failed_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_event(
2,
EventBody::StageFailed(failed_props(10, false)),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.duration_ms, Some(10));
assert_eq!(stage.state, Some(StageState::Failed));
}
#[test]
fn stage_retrying_sets_retrying_state() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_event(
2,
EventBody::StageFailed(failed_props(10, true)),
Some("build"),
))
.unwrap();
state
.apply_event(&test_event(
3,
EventBody::StageRetrying(retrying_props()),
Some("build"),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.state, Some(StageState::Retrying));
}
#[test]
fn stage_started_after_retrying_returns_to_running_and_resets_attempt_data() {
let mut state = RunProjection::default();
let stage_id = StageId::new("build", 1);
state
.apply_event(&test_stage_event(
1,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
state
.apply_event(&test_event(
2,
EventBody::StageFailed(failed_props(10, true)),
Some("build"),
))
.unwrap();
state
.apply_event(&test_event(
3,
EventBody::StageRetrying(retrying_props()),
Some("build"),
))
.unwrap();
state
.apply_event(&test_stage_event(
4,
EventBody::StageStarted(started_props()),
stage_id.clone(),
))
.unwrap();
let stage = state.stage(&stage_id).unwrap();
assert_eq!(stage.state, Some(StageState::Running));
// Prior attempt's terminal data must not leak into the new attempt.
assert!(stage.completion.is_none());
assert_eq!(stage.duration_ms, None);
}
}

285
lib/crates/fabro-store/src/slate/run_store.rs
View file

@ -12,7 +12,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream;
use super::blob_store::BlobStore;
use crate::run_state::{EventProjectionCache, RunProjectionReducer, build_summary};
use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, keys};
use crate::{Error, EventEnvelope, EventPayload, Result, RunProjection, StageId, keys};
const DEFAULT_EVENT_TAIL_LIMIT: usize = 1024;
#[derive(Clone)]
@ -213,6 +213,30 @@ impl RunDatabase {
list_events_from_with_limit(&self.inner.db, &self.inner.run_id, start_seq, limit).await
}
/// Returns up to `limit + 1` events for the given stage visit,
/// starting at `start_seq`. The `+1` lets callers compute `has_more`.
///
/// Implementation note: scans the unbounded run-event prefix and
/// filters by stage identity *before* applying `limit`, so a stage with
/// matches sparsely scattered late in the event log still returns its
/// full slice (no premature truncation from a generic `limit`-bounded
/// scan).
pub async fn list_events_for_stage_from_with_limit(
&self,
stage_id: &StageId,
start_seq: u32,
limit: usize,
) -> Result<Vec<EventEnvelope>> {
list_events_for_stage_from_with_limit(
&self.inner.db,
&self.inner.run_id,
stage_id,
start_seq,
limit,
)
.await
}
pub fn watch_events_from(
&self,
seq: u32,
@ -348,6 +372,74 @@ where
Ok(events)
}
async fn list_events_for_stage_from_with_limit<R>(
db: &R,
run_id: &RunId,
stage_id: &StageId,
start_seq: u32,
limit: usize,
) -> Result<Vec<EventEnvelope>>
where
R: DbRead + Sync,
{
// Unbounded scan first: filtering by stage identity with a generic
// limit-bounded scan would silently drop matches whenever the stage's
// events are sparse late in the event log.
//
// We probe just the stage identity fields with a small partial deserialize and
// only run the full `RunEvent` parse on matches. Most events in a run
// belong to other nodes, so this avoids deserializing large payloads
// (`agent.tool.completed.output`, `agent.message.text`, …) we'd discard.
#[derive(serde::Deserialize)]
struct StageIdProbe<'a> {
#[serde(default, borrow)]
stage_id: Option<&'a str>,
#[serde(default, borrow)]
node_id: Option<&'a str>,
}
let stage_id_string = stage_id.to_string();
let max_events = limit.saturating_add(1);
let mut iter = db.scan_prefix(keys::run_events_prefix(run_id)).await?;
let mut events: Vec<EventEnvelope> = Vec::new();
while let Some(entry) = iter.next().await? {
let key = key_to_string(&entry.key)?;
let Some(seq) = keys::parse_event_seq(&key) else {
continue;
};
if seq < start_seq {
continue;
}
let probe: StageIdProbe = serde_json::from_slice(&entry.value)?;
let matches_stage_id = probe.stage_id == Some(stage_id_string.as_str());
let matches_legacy_node_id = probe.stage_id.is_none()
&& stage_id.visit() == 1
&& probe.node_id == Some(stage_id.node_id());
if !matches_stage_id && !matches_legacy_node_id {
continue;
}
let event: RunEvent = serde_json::from_slice(&entry.value)?;
let envelope = EventEnvelope { seq, event };
if events.len() < max_events {
events.push(envelope);
continue;
}
if let Some((max_index, max_seq)) = events
.iter()
.enumerate()
.max_by_key(|(_, existing)| existing.seq)
.map(|(index, existing)| (index, existing.seq))
{
if seq < max_seq {
events[max_index] = envelope;
}
}
}
events.sort_by_key(|event| event.seq);
Ok(events)
}
async fn list_blobs<R>(db: &R) -> Result<Vec<RunBlobId>>
where
R: DbRead + Sync,
@ -375,9 +467,12 @@ mod tests {
use std::sync::Arc;
use std::time::Duration;
use fabro_types::{RunId, StageId};
use object_store::memory::InMemory;
use serde_json::json;
use crate::{Database, EventPayload};
use crate::Database;
#[tokio::test]
async fn list_blobs_reads_global_cas_namespace() {
let object_store = Arc::new(InMemory::new());
@ -394,4 +489,190 @@ mod tests {
assert_eq!(blob_ids, vec![first_id, second_id]);
}
fn stage_prompt_payload(run_id: &RunId, idx: u32, node_id: Option<&str>) -> EventPayload {
stage_prompt_payload_for_stage(run_id, idx, node_id, None)
}
fn stage_prompt_payload_for_stage(
run_id: &RunId,
idx: u32,
node_id: Option<&str>,
stage_id: Option<&StageId>,
) -> EventPayload {
let mut value = json!({
"id": format!("evt-{idx}"),
"ts": "2026-04-09T12:00:00Z",
"run_id": run_id.to_string(),
"event": "stage.prompt",
"properties": {
"visit": 1,
"text": format!("prompt {idx}"),
},
});
if let Some(node_id) = node_id {
value
.as_object_mut()
.unwrap()
.insert("node_id".into(), json!(node_id));
}
if let Some(stage_id) = stage_id {
value
.as_object_mut()
.unwrap()
.insert("stage_id".into(), json!(stage_id.to_string()));
}
EventPayload::new(value, run_id).unwrap()
}
async fn fresh_run() -> super::RunDatabase {
let object_store = Arc::new(InMemory::new());
let store = Database::new(object_store, "", Duration::from_millis(1), None);
let run_id: RunId = "01JT56VE4Z5NZ814GZN2JZD65A".parse().unwrap();
store.create_run(&run_id).await.unwrap()
}
#[tokio::test]
async fn list_events_for_stage_returns_only_matching_events_in_seq_order() {
let run = fresh_run().await;
let run_id = run.run_id();
run.append_event(&stage_prompt_payload(&run_id, 1, Some("alpha")))
.await
.unwrap();
run.append_event(&stage_prompt_payload(&run_id, 2, Some("beta")))
.await
.unwrap();
run.append_event(&stage_prompt_payload(&run_id, 3, Some("alpha")))
.await
.unwrap();
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![1, 3]);
}
#[tokio::test]
async fn list_events_for_stage_skips_events_with_no_stage_identity() {
let run = fresh_run().await;
let run_id = run.run_id();
run.append_event(&stage_prompt_payload(&run_id, 1, None))
.await
.unwrap();
run.append_event(&stage_prompt_payload(&run_id, 2, Some("alpha")))
.await
.unwrap();
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![2]);
}
#[tokio::test]
async fn list_events_for_stage_paginates_via_start_seq_on_filtered_slice() {
let run = fresh_run().await;
let run_id = run.run_id();
for idx in 1..=5 {
let node = if idx % 2 == 0 { "beta" } else { "alpha" };
run.append_event(&stage_prompt_payload(&run_id, idx, Some(node)))
.await
.unwrap();
}
// alpha events live at seqs 1, 3, 5. Start at seq=2 should skip seq=1.
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 2, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![3, 5]);
}
#[tokio::test]
async fn list_events_for_stage_walks_past_unrelated_events_for_sparse_matches() {
let run = fresh_run().await;
let run_id = run.run_id();
// 200 unrelated events first.
for idx in 1..=200 {
run.append_event(&stage_prompt_payload(&run_id, idx, Some("noise")))
.await
.unwrap();
}
// Then 3 sparse "alpha" events at the tail.
for idx in 201..=203 {
run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha")))
.await
.unwrap();
}
// limit smaller than the number of unrelated events would have
// truncated the upstream scan if we had post-filtered.
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 5)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![201, 202, 203]);
}
#[tokio::test]
async fn list_events_for_stage_returns_limit_plus_one_for_has_more_signal() {
let run = fresh_run().await;
let run_id = run.run_id();
for idx in 1..=5 {
run.append_event(&stage_prompt_payload(&run_id, idx, Some("alpha")))
.await
.unwrap();
}
let events = run
.list_events_for_stage_from_with_limit(&StageId::new("alpha", 1), 1, 2)
.await
.unwrap();
// With limit=2, we expect up to limit+1 = 3 envelopes so the
// caller can compute has_more.
assert_eq!(events.len(), 3);
}
#[tokio::test]
async fn list_events_for_stage_prefers_stage_id_over_node_id() {
let run = fresh_run().await;
let run_id = run.run_id();
let first_visit = StageId::new("verify", 1);
let second_visit = StageId::new("verify", 2);
run.append_event(&stage_prompt_payload_for_stage(
&run_id,
1,
Some("verify"),
Some(&first_visit),
))
.await
.unwrap();
run.append_event(&stage_prompt_payload_for_stage(
&run_id,
2,
Some("verify"),
Some(&second_visit),
))
.await
.unwrap();
let events = run
.list_events_for_stage_from_with_limit(&second_visit, 1, 100)
.await
.unwrap();
let seqs: Vec<u32> = events.iter().map(|e| e.seq).collect();
assert_eq!(seqs, vec![2]);
}
}

34
lib/crates/fabro-store/tests/serializable_projection.rs
View file

@ -5,8 +5,8 @@ use fabro_store::{RunProjection, SerializableProjection, StageId};
use fabro_types::graph::Graph;
use fabro_types::run::RunSpec;
use fabro_types::{
Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome, StartRecord,
TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
BilledModelUsage, Checkpoint, RunStatus, SandboxRecord, StageCompletion, StageOutcome,
StartRecord, TerminalStatus, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
@ -52,6 +52,28 @@ fn sample_checkpoint() -> Checkpoint {
}
}
fn sample_usage() -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": "gpt-5.2"
},
"tokens": {
"input_tokens": 123,
"output_tokens": 45
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": 168
}))
.expect("sample usage should deserialize")
}
#[test]
fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
let stage_id = StageId::new("build", 2);
@ -94,11 +116,17 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
stage.script_invocation = Some(json!({ "command": "cargo test" }));
stage.script_timing = Some(json!({ "duration_ms": 10 }));
stage.parallel_results = Some(json!([{ "stage": "fanout@1" }]));
stage.duration_ms = Some(1234);
stage.usage = Some(sample_usage());
stage.stdout = Some("stdout".to_string());
stage.stderr = Some("stderr".to_string());
let serialized = serde_json::to_value(SerializableProjection(&projection))
.expect("projection should serialize");
assert!(
serialized["stages"]["build@2"].get("usage").is_none(),
"stage usage is server-internal and should not be serialized"
);
let round_tripped: RunProjection =
serde_json::from_value(serialized).expect("serialized projection should deserialize");
let node = round_tripped.stage(&stage_id).expect("node should remain");
@ -138,6 +166,8 @@ fn serializable_projection_round_trips_and_trims_bulky_node_fields() {
node.parallel_results,
Some(json!([{ "stage": "fanout@1" }]))
);
assert_eq!(node.duration_ms, Some(1234));
assert_eq!(node.usage, None);
}
#[test]

4
lib/crates/fabro-types/src/lib.rs
View file

@ -73,7 +73,7 @@ pub use run::{
pub use run_blob_id::RunBlobId;
pub use run_event::{
EventBody, ExecOutputTail, InterviewOption, MetadataSnapshotFailureKind, MetadataSnapshotPhase,
RunEvent, RunNoticeLevel,
RunEvent, RunNoticeCode, RunNoticeLevel,
};
pub use run_id::{RunId, fixtures};
pub use run_projection::{PendingInterviewRecord, RunProjection, StageProjection, first_event_seq};
@ -81,7 +81,7 @@ pub use run_summary::RunSummary;
pub use sandbox_record::SandboxRecord;
pub use secret::{SecretMetadata, SecretType};
pub use stage_completion::StageCompletion;
pub use stage_id::{ParallelBranchId, StageId};
pub use stage_id::{InvalidStageVisit, ParallelBranchId, StageId};
pub use start::StartRecord;
pub use status::{
BlockedReason, FailureReason, InvalidTransition, ParseFailureReasonError,

25
lib/crates/fabro-types/src/outcome.rs
View file

@ -138,12 +138,32 @@ impl From<StageOutcome> for StageState {
match outcome {
StageOutcome::Succeeded => Self::Succeeded,
StageOutcome::PartiallySucceeded => Self::PartiallySucceeded,
StageOutcome::Failed { .. } => Self::Failed,
StageOutcome::Failed {
retry_requested: true,
} => Self::Retrying,
StageOutcome::Failed {
retry_requested: false,
} => Self::Failed,
StageOutcome::Skipped => Self::Skipped,
}
}
}
#[cfg(test)]
mod stage_state_tests {
use super::{StageOutcome, StageState};
#[test]
fn retry_requested_failure_projects_as_retrying() {
assert_eq!(
StageState::from(StageOutcome::Failed {
retry_requested: true,
}),
StageState::Retrying
);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FailureCategory {
@ -340,9 +360,10 @@ mod tests {
StageState::from(StageOutcome::Failed {
retry_requested: true,
}),
StageState::Failed
StageState::Retrying
);
assert!(StageState::Cancelled.is_terminal());
assert!(!StageState::Retrying.is_terminal());
assert!(!StageState::Running.is_terminal());
}
}

53
lib/crates/fabro-types/src/run_event/infra.rs
View file

@ -1,17 +1,48 @@
use serde::{Deserialize, Serialize};
/// Legacy `run.notice` codes paired with the new `metadata.snapshot.failed`
/// event for backward compatibility. Display layers suppress these so the
/// typed event renders without a duplicate raw warning.
pub const NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED: &str = "checkpoint_metadata_write_failed";
pub const NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED: &str = "checkpoint_metadata_push_failed";
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
Hash,
Serialize,
Deserialize,
strum::Display,
strum::EnumString,
strum::IntoStaticStr,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum RunNoticeCode {
ArtifactCollectionFailed,
ArtifactOffloadFailed,
ArtifactSyncFailed,
ArtifactUploadFailed,
CheckpointMetadataDegraded,
CheckpointMetadataPushFailed,
CheckpointMetadataWriteFailed,
DirtyWorktree,
GitDiffFailed,
GitPushFailed,
GithubTokenFailed,
ParallelBaseCheckpointFailed,
PullRequestFailed,
SandboxCleanupFailed,
SandboxGitUnavailable,
SandboxPreserved,
WorktreeSkippedNoGit,
}
#[must_use]
pub fn is_metadata_snapshot_compat_notice_code(code: &str) -> bool {
matches!(
code,
NOTICE_CODE_CHECKPOINT_METADATA_WRITE_FAILED | NOTICE_CODE_CHECKPOINT_METADATA_PUSH_FAILED
)
impl RunNoticeCode {
#[must_use]
pub fn is_metadata_snapshot_compat(self) -> bool {
matches!(
self,
Self::CheckpointMetadataWriteFailed | Self::CheckpointMetadataPushFailed
)
}
}
#[derive(

14
lib/crates/fabro-types/src/run_event/misc.rs
View file

@ -239,6 +239,20 @@ pub struct AgentCliCompletedProps {
pub duration_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AgentCliCancelledProps {
pub stdout: String,
pub stderr: String,
pub duration_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AgentCliTimedOutProps {
pub stdout: String,
pub stderr: String,
pub duration_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PullRequestCreatedProps {
pub pr_url: String,

10
lib/crates/fabro-types/src/run_event/mod.rs
View file

@ -260,6 +260,10 @@ pub enum EventBody {
AgentCliStarted(AgentCliStartedProps),
#[serde(rename = "agent.cli.completed")]
AgentCliCompleted(AgentCliCompletedProps),
#[serde(rename = "agent.cli.cancelled")]
AgentCliCancelled(AgentCliCancelledProps),
#[serde(rename = "agent.cli.timed_out")]
AgentCliTimedOut(AgentCliTimedOutProps),
#[serde(rename = "pull_request.created")]
PullRequestCreated(PullRequestCreatedProps),
#[serde(rename = "pull_request.failed")]
@ -445,6 +449,8 @@ impl EventBody {
Self::CommandCompleted(_) => "command.completed",
Self::AgentCliStarted(_) => "agent.cli.started",
Self::AgentCliCompleted(_) => "agent.cli.completed",
Self::AgentCliCancelled(_) => "agent.cli.cancelled",
Self::AgentCliTimedOut(_) => "agent.cli.timed_out",
Self::PullRequestCreated(_) => "pull_request.created",
Self::PullRequestFailed(_) => "pull_request.failed",
Self::DevcontainerResolved(_) => "devcontainer.resolved",
@ -1382,7 +1388,7 @@ mod tests {
for body in [
EventBody::RunNotice(RunNoticeProps {
level: RunNoticeLevel::Warn,
code: "git_diff_failed".to_string(),
code: RunNoticeCode::GitDiffFailed.to_string(),
message: "git diff failed".to_string(),
exec_output_tail: Some(tail.clone()),
}),
@ -1418,7 +1424,7 @@ mod tests {
for body in [
EventBody::RunNotice(RunNoticeProps {
level: RunNoticeLevel::Warn,
code: "git_diff_failed".to_string(),
code: RunNoticeCode::GitDiffFailed.to_string(),
message: "git diff failed".to_string(),
exec_output_tail: None,
}),

2
lib/crates/fabro-types/src/run_event/stage.rs
View file

@ -57,6 +57,8 @@ pub struct StageFailedProps {
pub will_retry: bool,
#[serde(default)]
pub duration_ms: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub billing: Option<BilledModelUsage>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

177
lib/crates/fabro-types/src/run_projection.rs
View file

@ -4,9 +4,9 @@ use std::num::NonZeroU32;
use chrono::{DateTime, Utc};
use crate::{
Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition, PullRequestRecord, Retro,
RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord, StageCompletion, StageId,
StartRecord,
BilledModelUsage, Checkpoint, Conclusion, InterviewQuestionRecord, InvalidTransition,
PullRequestRecord, Retro, RunControlAction, RunId, RunSpec, RunStatus, SandboxRecord,
StageCompletion, StageId, StageState, StartRecord,
};
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
@ -61,6 +61,17 @@ pub struct StageProjection {
pub live_streaming: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub termination: Option<crate::CommandTermination>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<DateTime<Utc>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub duration_ms: Option<u64>,
/// Server-internal billing usage for the latest attempt; not part of the
/// wire contract because `BilledModelUsage` is not modeled in OpenAPI.
/// Read only in-process by the billing handler.
#[serde(skip)]
pub usage: Option<BilledModelUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub state: Option<StageState>,
}
/// Convert a 1-based event sequence number into the `NonZeroU32` form used for
@ -78,6 +89,8 @@ impl StageProjection {
prompt: None,
response: None,
completion: None,
duration_ms: None,
usage: None,
provider_used: None,
diff: None,
script_invocation: None,
@ -90,8 +103,55 @@ impl StageProjection {
streams_separated: None,
live_streaming: None,
termination: None,
started_at: None,
state: None,
}
}
/// Effective lifecycle state derived from stored event data.
///
/// Falls back to deriving from `completion` for projections that predate
/// the stored `state` field, so old serialized projections still work
/// without a backfill.
#[must_use]
pub fn effective_state(&self) -> StageState {
self.state.unwrap_or_else(|| match &self.completion {
Some(completion) => StageState::from(completion.outcome),
None => StageState::Running,
})
}
/// Live wall-clock runtime in seconds.
///
/// While the stage is non-terminal (`Pending`, `Running`, or `Retrying`),
/// this returns the elapsed time since `started_at` so the UI can tick
/// client-side. Once terminal, the stored `duration_ms` is returned. This
/// also handles retries safely: a new `StageStarted` resets the state
/// back to `Running` and keeps the live computation correct even if a
/// previous attempt left a stale `duration_ms`.
#[must_use]
pub fn runtime_secs(&self, now: DateTime<Utc>) -> Option<f64> {
let state = self.effective_state();
if matches!(
state,
StageState::Running | StageState::Retrying | StageState::Pending
) {
return self.started_at.map(|started| {
now.signed_duration_since(started).num_milliseconds().max(0) as f64 / 1000.0
});
}
self.duration_ms.map(|ms| ms as f64 / 1000.0)
}
/// Begin a new attempt (or visit) for this stage: clear every
/// per-attempt field so prior-attempt data does not leak, then record
/// `started_at` and `state = Running`. Preserves `first_event_seq`
/// (identity / sort key).
pub fn begin_attempt(&mut self, started_at: DateTime<Utc>) {
*self = Self::new(self.first_event_seq);
self.started_at = Some(started_at);
self.state = Some(StageState::Running);
}
}
impl RunProjection {
@ -99,12 +159,32 @@ impl RunProjection {
self.stages.get(stage)
}
/// Iterate stages in `first_event_seq` order (the chronological order in
/// which each stage's first lifecycle event was recorded). Internal
/// storage is a `HashMap`, so iteration would otherwise be
/// non-deterministic; every caller wants chronological order, so we sort
/// here once instead of asking each caller to remember.
pub fn iter_stages(&self) -> impl Iterator<Item = (&StageId, &StageProjection)> {
self.stages.iter()
let mut entries: Vec<(&StageId, &StageProjection)> = self.stages.iter().collect();
entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| {
left_stage
.first_event_seq
.cmp(&right_stage.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
entries.into_iter()
}
/// Mutable counterpart of [`iter_stages`]. Same chronological ordering.
pub fn iter_stages_mut(&mut self) -> impl Iterator<Item = (&StageId, &mut StageProjection)> {
self.stages.iter_mut()
let mut entries: Vec<(&StageId, &mut StageProjection)> = self.stages.iter_mut().collect();
entries.sort_by(|(left_id, left_stage), (right_id, right_stage)| {
left_stage
.first_event_seq
.cmp(&right_stage.first_event_seq)
.then_with(|| left_id.cmp(right_id))
});
entries.into_iter()
}
pub fn is_empty(&self) -> bool {
@ -186,3 +266,90 @@ impl RunProjection {
}
}
}
#[cfg(test)]
mod iter_stages_tests {
use std::num::NonZeroU32;
use super::RunProjection;
fn seq(n: u32) -> NonZeroU32 {
NonZeroU32::new(n).unwrap()
}
#[test]
fn iter_stages_yields_chronological_order_across_nodes() {
let mut p = RunProjection::default();
// Insert in non-monotonic seq order to exercise the sort.
p.stage_entry("c", 1, seq(30));
p.stage_entry("a", 1, seq(10));
p.stage_entry("b", 1, seq(20));
let order: Vec<&str> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.node_id())
.collect();
assert_eq!(order, vec!["a", "b", "c"]);
}
#[test]
fn iter_stages_orders_visits_within_a_node() {
let mut p = RunProjection::default();
// Visit 2 inserted first; visit 1's earlier first_event_seq must still
// win the chronological ordering.
p.stage_entry("verify", 2, seq(50));
p.stage_entry("verify", 1, seq(20));
let visits: Vec<u32> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.visit())
.collect();
assert_eq!(visits, vec![1, 2]);
}
#[test]
fn iter_stages_mut_yields_chronological_order() {
let mut p = RunProjection::default();
p.stage_entry("c", 1, seq(30));
p.stage_entry("a", 1, seq(10));
p.stage_entry("b", 1, seq(20));
let order: Vec<String> = p
.iter_stages_mut()
.map(|(stage_id, _)| stage_id.node_id().to_string())
.collect();
assert_eq!(order, vec!["a", "b", "c"]);
}
#[test]
fn iter_stages_tie_breaks_same_first_event_seq_by_stage_id() {
for _ in 0..128 {
let mut p = RunProjection::default();
p.stage_entry("verify", 2, seq(10));
p.stage_entry("build", 1, seq(10));
p.stage_entry("verify", 1, seq(10));
let order: Vec<String> = p
.iter_stages()
.map(|(stage_id, _)| stage_id.to_string())
.collect();
assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]);
}
}
#[test]
fn iter_stages_mut_tie_breaks_same_first_event_seq_by_stage_id() {
for _ in 0..128 {
let mut p = RunProjection::default();
p.stage_entry("verify", 2, seq(10));
p.stage_entry("build", 1, seq(10));
p.stage_entry("verify", 1, seq(10));
let order: Vec<String> = p
.iter_stages_mut()
.map(|(stage_id, _)| stage_id.to_string())
.collect();
assert_eq!(order, vec!["build@1", "verify@1", "verify@2"]);
}
}
}

39
lib/crates/fabro-types/src/stage_id.rs
View file

@ -1,4 +1,5 @@
use std::fmt;
use std::num::NonZeroU32;
use std::str::FromStr;
use serde::de::Error as _;
@ -7,16 +8,21 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct StageId {
node_id: String,
visit: u32,
visit: NonZeroU32,
}
impl StageId {
#[must_use]
pub fn new(node_id: impl Into<String>, visit: u32) -> Self {
Self {
Self::try_new(node_id, visit).expect("stage id visit must be greater than zero")
}
pub fn try_new(node_id: impl Into<String>, visit: u32) -> Result<Self, InvalidStageVisit> {
let visit = NonZeroU32::new(visit).ok_or(InvalidStageVisit)?;
Ok(Self {
node_id: node_id.into(),
visit,
}
})
}
#[must_use]
@ -26,7 +32,7 @@ impl StageId {
#[must_use]
pub fn visit(&self) -> u32 {
self.visit
self.visit.get()
}
}
@ -47,6 +53,17 @@ impl fmt::Display for ParseStageIdError {
impl std::error::Error for ParseStageIdError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InvalidStageVisit;
impl fmt::Display for InvalidStageVisit {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("stage id visit must be greater than zero")
}
}
impl std::error::Error for InvalidStageVisit {}
impl FromStr for StageId {
type Err = ParseStageIdError;
@ -67,7 +84,7 @@ impl FromStr for StageId {
let visit = visit
.parse()
.map_err(|err| ParseStageIdError(format!("invalid stage id visit: {err}")))?;
Ok(Self::new(node_id, visit))
Self::try_new(node_id, visit).map_err(|err| ParseStageIdError(err.to_string()))
}
}
@ -224,6 +241,18 @@ mod tests {
assert!(err.to_string().starts_with("invalid stage id visit:"));
}
#[test]
fn parse_rejects_zero_visit() {
let err = "code@0".parse::<StageId>().unwrap_err();
assert_eq!(err.to_string(), "stage id visit must be greater than zero");
}
#[test]
fn try_new_rejects_zero_visit() {
let err = StageId::try_new("code", 0).unwrap_err();
assert_eq!(err.to_string(), "stage id visit must be greater than zero");
}
#[test]
fn parse_rejects_empty_node_id() {
let err = "@3".parse::<StageId>().unwrap_err();

291
lib/crates/fabro-workflow/src/billing_rollup.rs Normal file
View file

@ -0,0 +1,291 @@
use std::collections::{BTreeMap, HashMap};
use fabro_types::{BilledModelUsage, BilledTokenCounts, RunProjection};
#[derive(Debug, Clone, PartialEq)]
pub struct ProjectionBillingStage {
pub node_id: String,
pub billing: BilledTokenCounts,
pub duration_ms: u64,
pub model_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProjectionBillingByModel {
pub model_id: String,
pub stages: i64,
pub billing: BilledTokenCounts,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct ProjectionBillingRollup {
pub stages: Vec<ProjectionBillingStage>,
pub totals: BilledTokenCounts,
pub by_model: Vec<ProjectionBillingByModel>,
pub runtime_ms: u64,
pub billed_visit_count: usize,
}
impl ProjectionBillingRollup {
#[must_use]
pub fn billing_if_present(&self) -> Option<BilledTokenCounts> {
(self.billed_visit_count > 0).then(|| self.totals.clone())
}
}
#[must_use]
pub fn billing_rollup_from_projection(projection: &RunProjection) -> ProjectionBillingRollup {
let mut stage_indices = HashMap::<String, usize>::new();
let mut stages = Vec::<ProjectionBillingStage>::new();
let mut by_model = BTreeMap::<String, ProjectionBillingByModel>::new();
let mut totals = BilledTokenCounts::default();
let mut runtime_ms = 0_u64;
let mut billed_visit_count = 0_usize;
for (stage_id, stage) in projection.iter_stages() {
if is_exit_stage(projection, stage_id.node_id()) {
continue;
}
if stage.completion.is_none() && stage.duration_ms.is_none() && stage.usage.is_none() {
continue;
}
let node_id = stage_id.node_id();
let index = *stage_indices.entry(node_id.to_string()).or_insert_with(|| {
let index = stages.len();
stages.push(ProjectionBillingStage {
node_id: node_id.to_string(),
billing: BilledTokenCounts::default(),
duration_ms: 0,
model_id: None,
});
index
});
let row = &mut stages[index];
if let Some(duration_ms) = stage.duration_ms {
row.duration_ms = row.duration_ms.saturating_add(duration_ms);
runtime_ms = runtime_ms.saturating_add(duration_ms);
}
if let Some(usage) = stage.usage.as_ref() {
billed_visit_count += 1;
row.model_id = Some(usage.model_id().to_string());
accumulate_usage(&mut row.billing, usage);
accumulate_usage(&mut totals, usage);
let model_id = usage.model_id().to_string();
let model_entry =
by_model
.entry(model_id.clone())
.or_insert_with(|| ProjectionBillingByModel {
model_id,
stages: 0,
billing: BilledTokenCounts::default(),
});
model_entry.stages += 1;
accumulate_usage(&mut model_entry.billing, usage);
}
}
ProjectionBillingRollup {
stages,
totals,
by_model: by_model.into_values().collect(),
runtime_ms,
billed_visit_count,
}
}
fn is_exit_stage(projection: &RunProjection, node_id: &str) -> bool {
projection
.spec()
.and_then(|spec| spec.graph().nodes.get(node_id))
.is_some_and(|node| node.handler_type() == Some("exit"))
}
fn accumulate_usage(counts: &mut BilledTokenCounts, usage: &BilledModelUsage) {
let tokens = usage.tokens();
counts.input_tokens += tokens.input_tokens;
counts.output_tokens += tokens.output_tokens;
counts.reasoning_tokens += tokens.reasoning_tokens;
counts.cache_read_tokens += tokens.cache_read_tokens;
counts.cache_write_tokens += tokens.cache_write_tokens;
counts.total_tokens += tokens.total_tokens();
if let Some(value) = usage.total_usd_micros {
*counts.total_usd_micros.get_or_insert(0) += value;
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use fabro_types::{
AttrValue, BilledModelUsage, Graph, Node, RunProjection, RunSpec, StageCompletion,
StageOutcome, WorkflowSettings, first_event_seq, fixtures,
};
use serde_json::json;
use super::billing_rollup_from_projection;
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn rollup_groups_stage_rows_by_node_and_sums_retry_visit_usage() {
let mut projection = RunProjection::default();
let failed_usage = test_usage("gpt-old", 100, 10);
let success_usage = test_usage("gpt-new", 200, 20);
let first = projection.stage_entry("verify", 1, first_event_seq(1));
first.duration_ms = Some(1200);
first.usage = Some(failed_usage);
first.completion = Some(StageCompletion {
outcome: StageOutcome::Failed {
retry_requested: true,
},
notes: None,
failure_reason: Some("try again".to_string()),
timestamp: chrono::Utc::now(),
});
let second = projection.stage_entry("verify", 2, first_event_seq(2));
second.duration_ms = Some(800);
second.usage = Some(success_usage);
second.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "verify");
assert_eq!(rollup.stages[0].model_id.as_deref(), Some("gpt-new"));
assert_eq!(rollup.stages[0].duration_ms, 2000);
assert_eq!(rollup.stages[0].billing.input_tokens, 300);
assert_eq!(rollup.stages[0].billing.output_tokens, 30);
assert_eq!(rollup.stages[0].billing.total_usd_micros, Some(330));
assert_eq!(rollup.runtime_ms, 2000);
assert_eq!(rollup.totals.input_tokens, 300);
assert_eq!(rollup.totals.output_tokens, 30);
assert_eq!(rollup.totals.total_usd_micros, Some(330));
assert_eq!(rollup.billed_visit_count, 2);
assert_eq!(rollup.by_model.len(), 2);
assert_eq!(rollup.by_model[0].model_id, "gpt-new");
assert_eq!(rollup.by_model[0].stages, 1);
assert_eq!(rollup.by_model[0].billing.input_tokens, 200);
assert_eq!(rollup.by_model[1].model_id, "gpt-old");
assert_eq!(rollup.by_model[1].stages, 1);
assert_eq!(rollup.by_model[1].billing.input_tokens, 100);
}
#[test]
fn rollup_includes_completed_non_llm_stage_rows_with_zero_billing() {
let mut projection = RunProjection::default();
let stage = projection.stage_entry("start", 1, first_event_seq(1));
stage.duration_ms = Some(25);
stage.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "start");
assert_eq!(rollup.stages[0].duration_ms, 25);
assert!(rollup.stages[0].model_id.is_none());
assert_eq!(rollup.stages[0].billing.input_tokens, 0);
assert_eq!(rollup.runtime_ms, 25);
assert!(rollup.by_model.is_empty());
assert!(rollup.billing_if_present().is_none());
}
#[test]
fn rollup_excludes_terminal_exit_stage_rows() {
let mut projection = RunProjection::default();
projection.spec = Some(run_spec_with_exit_node());
let start = projection.stage_entry("start", 1, first_event_seq(1));
start.duration_ms = Some(25);
start.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let exit = projection.stage_entry("exit", 1, first_event_seq(2));
exit.duration_ms = Some(7);
exit.completion = Some(StageCompletion {
outcome: StageOutcome::Succeeded,
notes: None,
failure_reason: None,
timestamp: chrono::Utc::now(),
});
let rollup = billing_rollup_from_projection(&projection);
assert_eq!(rollup.stages.len(), 1);
assert_eq!(rollup.stages[0].node_id, "start");
assert_eq!(rollup.runtime_ms, 25);
}
fn run_spec_with_exit_node() -> RunSpec {
let mut graph = Graph::new("test");
graph.nodes.insert("start".to_string(), {
let mut node = Node::new("start");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Mdiamond".to_string()),
);
node
});
graph.nodes.insert("exit".to_string(), {
let mut node = Node::new("exit");
node.attrs.insert(
"shape".to_string(),
AttrValue::String("Msquare".to_string()),
);
node
});
RunSpec {
run_id: fixtures::RUN_1,
settings: WorkflowSettings::default(),
graph,
workflow_slug: None,
source_directory: None,
labels: HashMap::new(),
provenance: None,
manifest_blob: None,
definition_blob: None,
git: None,
fork_source_ref: None,
in_place: false,
}
}
}

126
lib/crates/fabro-workflow/src/devcontainer_bridge.rs
View file

@ -1,5 +1,3 @@
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::Instant;
use fabro_agent::sandbox::Sandbox;
@ -7,10 +5,10 @@ use fabro_devcontainer::DevcontainerSpec;
use fabro_sandbox::daytona::{DaytonaSnapshotConfig, DockerfileSource};
use futures::future::try_join_all;
use sha2::{Digest, Sha256};
use tokio_util::sync::CancellationToken;
use crate::error::Error;
use crate::event::{Emitter, Event};
use crate::handler::sandbox_cancel_token;
/// Compute a deterministic snapshot name from Dockerfile content.
pub fn snapshot_name_for_dockerfile(dockerfile: &str) -> String {
@ -39,7 +37,7 @@ pub async fn run_devcontainer_lifecycle(
phase: &str,
commands: &[fabro_devcontainer::Command],
timeout_ms: u64,
cancel_requested: Option<Arc<AtomicBool>>,
cancel_token: CancellationToken,
) -> Result<(), Error> {
if commands.is_empty() {
return Ok(());
@ -61,7 +59,7 @@ pub async fn run_devcontainer_lifecycle(
&format!("sh -c {}", shlex::try_quote(s).unwrap_or_else(|_| s.into())),
index,
timeout_ms,
cancel_requested.clone(),
cancel_token.clone(),
)
.await?;
}
@ -78,7 +76,7 @@ pub async fn run_devcontainer_lifecycle(
&joined,
index,
timeout_ms,
cancel_requested.clone(),
cancel_token.clone(),
)
.await?;
}
@ -92,7 +90,7 @@ pub async fn run_devcontainer_lifecycle(
);
let phase = phase.to_string();
let name = name.clone();
let cancel_requested = cancel_requested.clone();
let cancel_token = cancel_token.clone();
async move {
let cmd_start = Instant::now();
emitter.emit(&Event::DevcontainerLifecycleCommandStarted {
@ -100,14 +98,14 @@ pub async fn run_devcontainer_lifecycle(
command: name.clone(),
index,
});
let cancel_token = sandbox_cancel_token(cancel_requested);
let child_token = cancel_token.child_token();
let result = sandbox
.exec_command(
&command,
timeout_ms,
None,
None,
cancel_token.clone(),
Some(child_token.clone()),
)
.await
.map_err(|e| {
@ -115,12 +113,10 @@ pub async fn run_devcontainer_lifecycle(
"Devcontainer {phase} parallel command '{name}' failed: {e}"
))
})?;
if let Some(token) = &cancel_token {
if token.is_cancelled() {
return Err(Error::Cancelled);
}
token.cancel();
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
child_token.cancel();
let cmd_duration = crate::millis_u64(cmd_start.elapsed());
if !result.is_success() {
let exit_code = result.display_exit_code();
@ -175,7 +171,7 @@ async fn run_single_lifecycle_command(
command: &str,
index: usize,
timeout_ms: u64,
cancel_requested: Option<Arc<AtomicBool>>,
cancel_token: CancellationToken,
) -> Result<(), Error> {
emitter.emit(&Event::DevcontainerLifecycleCommandStarted {
phase: phase.to_string(),
@ -183,19 +179,17 @@ async fn run_single_lifecycle_command(
index,
});
let cmd_start = Instant::now();
let cancel_token = sandbox_cancel_token(cancel_requested);
let child_token = cancel_token.child_token();
let result = sandbox
.exec_command(command, timeout_ms, None, None, cancel_token.clone())
.exec_command(command, timeout_ms, None, None, Some(child_token.clone()))
.await
.map_err(|e| {
Error::engine_with_source(format!("Devcontainer {phase} command failed"), &e)
})?;
if let Some(token) = &cancel_token {
if token.is_cancelled() {
return Err(Error::Cancelled);
}
token.cancel();
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
child_token.cancel();
let cmd_duration = crate::millis_u64(cmd_start.elapsed());
if !result.is_success() {
let exit_code = result.display_exit_code();
@ -227,7 +221,6 @@ async fn run_single_lifecycle_command(
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
@ -437,9 +430,16 @@ mod tests {
let sandbox = TestSandbox::new();
let emitter = Emitter::default();
let commands = vec![fabro_devcontainer::Command::Shell("echo hi".to_string())];
run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None)
.await
.unwrap();
run_devcontainer_lifecycle(
&sandbox,
&emitter,
"on_create",
&commands,
300_000,
CancellationToken::new(),
)
.await
.unwrap();
let captured = sandbox.captured_commands();
assert_eq!(captured.len(), 1);
assert!(captured[0].contains("echo hi"), "command: {}", captured[0]);
@ -453,9 +453,16 @@ mod tests {
"echo".to_string(),
"hi".to_string(),
])];
run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None)
.await
.unwrap();
run_devcontainer_lifecycle(
&sandbox,
&emitter,
"on_create",
&commands,
300_000,
CancellationToken::new(),
)
.await
.unwrap();
let captured = sandbox.captured_commands();
assert_eq!(captured.len(), 1);
assert!(
@ -475,9 +482,16 @@ mod tests {
});
let sandbox = TestSandbox::new();
let commands = vec![fabro_devcontainer::Command::Shell("echo hi".to_string())];
run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None)
.await
.unwrap();
run_devcontainer_lifecycle(
&sandbox,
&emitter,
"on_create",
&commands,
300_000,
CancellationToken::new(),
)
.await
.unwrap();
let events = events.lock().unwrap();
let started = events[0].properties().unwrap();
assert_eq!(events[0].event_name(), "devcontainer.lifecycle.started");
@ -515,9 +529,15 @@ mod tests {
});
let sandbox = TestSandbox::with_exit_code(1);
let commands = vec![fabro_devcontainer::Command::Shell("false".to_string())];
let result =
run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &commands, 300_000, None)
.await;
let result = run_devcontainer_lifecycle(
&sandbox,
&emitter,
"on_create",
&commands,
300_000,
CancellationToken::new(),
)
.await;
assert!(result.is_err());
let events = events.lock().unwrap();
let failed = events
@ -550,9 +570,16 @@ mod tests {
events_clone.lock().unwrap().push(event.clone());
});
let sandbox = TestSandbox::new();
run_devcontainer_lifecycle(&sandbox, &emitter, "on_create", &[], 300_000, None)
.await
.unwrap();
run_devcontainer_lifecycle(
&sandbox,
&emitter,
"on_create",
&[],
300_000,
CancellationToken::new(),
)
.await
.unwrap();
assert!(events.lock().unwrap().is_empty());
}
@ -564,9 +591,16 @@ mod tests {
map.insert("install".to_string(), "npm install".to_string());
map.insert("build".to_string(), "npm run build".to_string());
let commands = vec![fabro_devcontainer::Command::Parallel(map)];
run_devcontainer_lifecycle(&sandbox, &emitter, "post_create", &commands, 300_000, None)
.await
.unwrap();
run_devcontainer_lifecycle(
&sandbox,
&emitter,
"post_create",
&commands,
300_000,
CancellationToken::new(),
)
.await
.unwrap();
let captured = sandbox.captured_commands();
assert_eq!(captured.len(), 2);
}
@ -576,7 +610,8 @@ mod tests {
let sandbox = TestSandbox::waiting_for_cancel();
let emitter = Emitter::default();
let commands = vec![fabro_devcontainer::Command::Shell("sleep 5".to_string())];
let cancel_requested = Arc::new(AtomicBool::new(true));
let cancel_token = CancellationToken::new();
cancel_token.cancel();
let result = run_devcontainer_lifecycle(
&sandbox,
@ -584,7 +619,7 @@ mod tests {
"on_create",
&commands,
300_000,
Some(cancel_requested),
cancel_token,
)
.await;
@ -600,7 +635,8 @@ mod tests {
map.insert("install".to_string(), "sleep 5".to_string());
map.insert("build".to_string(), "sleep 5".to_string());
let commands = vec![fabro_devcontainer::Command::Parallel(map)];
let cancel_requested = Arc::new(AtomicBool::new(true));
let cancel_token = CancellationToken::new();
cancel_token.cancel();
let result = run_devcontainer_lifecycle(
&sandbox,
@ -608,7 +644,7 @@ mod tests {
"post_create",
&commands,
300_000,
Some(cancel_requested),
cancel_token,
)
.await;

1
lib/crates/fabro-workflow/src/error.rs
View file

@ -1838,6 +1838,7 @@ mod tests {
failure: failure.clone(),
will_retry: false,
duration_ms: 0,
billing: None,
actor: None,
};

2
lib/crates/fabro-workflow/src/event.rs
View file

@ -8,7 +8,7 @@ mod stored_fields;
#[cfg(test)]
mod test_support;
pub use fabro_types::{EventBody, RunNoticeLevel};
pub use fabro_types::{EventBody, RunNoticeCode, RunNoticeLevel};
pub use self::convert::{to_run_event, to_run_event_at};
pub use self::emitter::Emitter;

97
lib/crates/fabro-workflow/src/event/convert.rs
View file

@ -290,12 +290,14 @@ fn event_body_from_event(event: &Event) -> EventBody {
failure,
will_retry,
duration_ms,
billing,
..
} => EventBody::StageFailed(fabro_types::StageFailedProps {
index: *index,
failure: Some(failure.clone()),
will_retry: *will_retry,
duration_ms: *duration_ms,
billing: billing.clone(),
}),
Event::StageRetrying {
index,
@ -1022,6 +1024,26 @@ fn event_body_from_event(event: &Event) -> EventBody {
count: *count,
})
}
Event::AgentCliCancelled {
stdout,
stderr,
duration_ms,
..
} => EventBody::AgentCliCancelled(fabro_types::AgentCliCancelledProps {
stdout: stdout.clone(),
stderr: stderr.clone(),
duration_ms: *duration_ms,
}),
Event::AgentCliTimedOut {
stdout,
stderr,
duration_ms,
..
} => EventBody::AgentCliTimedOut(fabro_types::AgentCliTimedOutProps {
stdout: stdout.clone(),
stderr: stderr.clone(),
duration_ms: *duration_ms,
}),
Event::PullRequestCreated {
pr_url,
pr_number,
@ -1183,8 +1205,8 @@ mod tests {
use std::collections::BTreeMap;
use ::fabro_types::{
EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeLevel, RunProvenance,
StageId, SystemActorKind, fixtures, run_event as fabro_types,
EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, RunNoticeLevel,
RunProvenance, StageId, SystemActorKind, fixtures, run_event as fabro_types,
};
use chrono::Utc;
use fabro_agent::{AgentEvent, SandboxEvent};
@ -1194,7 +1216,7 @@ mod tests {
use crate::error::Error;
use crate::event::test_support::user_principal;
use crate::event::{Event, StageScope};
use crate::outcome::FailureDetail;
use crate::outcome::{BilledModelUsage, FailureDetail};
#[derive(Debug)]
struct EventTestCause;
@ -1216,6 +1238,28 @@ mod tests {
}
}
fn test_usage(model_id: &str, input_tokens: i64, output_tokens: i64) -> BilledModelUsage {
serde_json::from_value(serde_json::json!({
"input": {
"usage": {
"model": {
"provider": "openai",
"model_id": model_id
},
"tokens": {
"input_tokens": input_tokens,
"output_tokens": output_tokens
}
},
"facts": {
"provider": "open_ai"
}
},
"total_usd_micros": input_tokens + output_tokens
}))
.unwrap()
}
#[test]
fn run_event_stage_completed_places_node_fields_in_header() {
let stored = to_run_event_at(
@ -1295,6 +1339,7 @@ mod tests {
#[test]
fn run_event_stage_failure_keeps_failure_detail() {
let usage = test_usage("gpt-5.2", 321, 54);
let stored = to_run_event(&fixtures::RUN_3, &Event::StageFailed {
node_id: "code".to_string(),
name: "Code".to_string(),
@ -1305,6 +1350,7 @@ mod tests {
),
will_retry: true,
duration_ms: 5000,
billing: Some(usage.clone()),
actor: None,
});
@ -1313,6 +1359,7 @@ mod tests {
assert_eq!(properties["failure"]["message"], "lint failed");
assert_eq!(properties["failure"]["failure_class"], "deterministic");
assert_eq!(properties["will_retry"], true);
assert_eq!(properties["billing"], serde_json::to_value(&usage).unwrap());
}
#[test]
@ -1632,7 +1679,7 @@ mod tests {
fn run_notice_maps_exec_output_tail_to_props() {
let stored = to_run_event(&fixtures::RUN_1, &Event::RunNotice {
level: RunNoticeLevel::Warn,
code: "git_diff_failed".to_string(),
code: RunNoticeCode::GitDiffFailed.to_string(),
message: "git diff failed".to_string(),
exec_output_tail: Some(exec_tail()),
});
@ -1825,6 +1872,48 @@ mod tests {
});
}
#[test]
fn agent_cli_cancelled_maps_to_event_body_with_node_id() {
let stored = to_run_event(&fixtures::RUN_1, &Event::AgentCliCancelled {
node_id: "code".to_string(),
stdout: "out".to_string(),
stderr: "err".to_string(),
duration_ms: 42,
});
assert_eq!(stored.event_name(), "agent.cli.cancelled");
assert_eq!(stored.node_id.as_deref(), Some("code"));
match &stored.body {
EventBody::AgentCliCancelled(props) => {
assert_eq!(props.stdout, "out");
assert_eq!(props.stderr, "err");
assert_eq!(props.duration_ms, 42);
}
other => panic!("expected AgentCliCancelled, got {other:?}"),
}
}
#[test]
fn agent_cli_timed_out_maps_to_event_body_with_node_id() {
let stored = to_run_event(&fixtures::RUN_1, &Event::AgentCliTimedOut {
node_id: "code".to_string(),
stdout: "out".to_string(),
stderr: "err".to_string(),
duration_ms: 99,
});
assert_eq!(stored.event_name(), "agent.cli.timed_out");
assert_eq!(stored.node_id.as_deref(), Some("code"));
match &stored.body {
EventBody::AgentCliTimedOut(props) => {
assert_eq!(props.stdout, "out");
assert_eq!(props.stderr, "err");
assert_eq!(props.duration_ms, 99);
}
other => panic!("expected AgentCliTimedOut, got {other:?}"),
}
}
#[test]
fn stall_watchdog_timeout_populates_watchdog_actor() {
let stored = to_run_event(&fixtures::RUN_1, &Event::StallWatchdogTimeout {

15
lib/crates/fabro-workflow/src/event/emitter.rs
View file

@ -1,7 +1,7 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicI64, Ordering};
use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeLevel};
use ::fabro_types::{ExecOutputTail, RunEvent, RunId, RunNoticeCode, RunNoticeLevel};
use chrono::Utc;
use fabro_agent::{WorktreeEvent, WorktreeEventCallback};
@ -76,15 +76,10 @@ impl Emitter {
self.emit_with_scope(event, Some(scope));
}
pub fn notice(
&self,
level: RunNoticeLevel,
code: impl Into<String>,
message: impl Into<String>,
) {
pub fn notice(&self, level: RunNoticeLevel, code: RunNoticeCode, message: impl Into<String>) {
self.emit(&Event::RunNotice {
level,
code: code.into(),
code: code.to_string(),
message: message.into(),
exec_output_tail: None,
});
@ -93,13 +88,13 @@ impl Emitter {
pub fn notice_with_tail(
&self,
level: RunNoticeLevel,
code: impl Into<String>,
code: RunNoticeCode,
message: impl Into<String>,
exec_output_tail: Option<ExecOutputTail>,
) {
self.emit(&Event::RunNotice {
level,
code: code.into(),
code: code.to_string(),
message: message.into(),
exec_output_tail,
});

27
lib/crates/fabro-workflow/src/event/events.rs
View file

@ -206,6 +206,7 @@ pub enum Event {
failure: FailureDetail,
will_retry: bool,
duration_ms: u64,
billing: Option<BilledModelUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
actor: Option<Principal>,
},
@ -560,6 +561,18 @@ pub enum Event {
exit_code: i32,
duration_ms: u64,
},
AgentCliCancelled {
node_id: String,
stdout: String,
stderr: String,
duration_ms: u64,
},
AgentCliTimedOut {
node_id: String,
stdout: String,
stderr: String,
duration_ms: u64,
},
PullRequestCreated {
pr_url: String,
pr_number: u64,
@ -1289,6 +1302,20 @@ impl Event {
Self::AgentSteerDropped { reason, count, .. } => {
warn!(?reason, count, "Steer dropped");
}
Self::AgentCliCancelled {
node_id,
duration_ms,
..
} => {
debug!(node_id, duration_ms, "Agent CLI cancelled");
}
Self::AgentCliTimedOut {
node_id,
duration_ms,
..
} => {
debug!(node_id, duration_ms, "Agent CLI timed out");
}
Self::PullRequestCreated {
pr_url,
pr_number,

2
lib/crates/fabro-workflow/src/event/names.rs
View file

@ -120,6 +120,8 @@ pub fn event_name(event: &Event) -> &'static str {
Event::AgentSteeringDetached { .. } => "agent.steering.detached",
Event::AgentSteerBuffered { .. } => "agent.steer.buffered",
Event::AgentSteerDropped { .. } => "agent.steer.dropped",
Event::AgentCliCancelled { .. } => "agent.cli.cancelled",
Event::AgentCliTimedOut { .. } => "agent.cli.timed_out",
Event::PullRequestCreated { .. } => "pull_request.created",
Event::PullRequestFailed { .. } => "pull_request.failed",
Event::DevcontainerResolved { .. } => "devcontainer.resolved",

4
lib/crates/fabro-workflow/src/event/stored_fields.rs
View file

@ -117,7 +117,9 @@ fn stored_event_fields_for_variant(event: &Event) -> StoredEventFields {
| Event::CommandStarted { node_id, .. }
| Event::CommandCompleted { node_id, .. }
| Event::AgentCliStarted { node_id, .. }
| Event::AgentCliCompleted { node_id, .. } => node_stored_fields(Some(node_id.clone())),
| Event::AgentCliCompleted { node_id, .. }
| Event::AgentCliCancelled { node_id, .. }
| Event::AgentCliTimedOut { node_id, .. } => node_stored_fields(Some(node_id.clone())),
Event::AgentSteeringAttached { node_id, visit }
| Event::AgentSteeringDetached { node_id, visit } => {
let node_id_str = node_id.clone();

15
lib/crates/fabro-workflow/src/handler/agent.rs
View file

@ -7,6 +7,7 @@ use fabro_agent::Sandbox;
use fabro_graphviz::graph::{Graph, Node};
use fabro_template::{TemplateContext, render as render_template};
use fabro_types::RunId;
use tokio_util::sync::CancellationToken;
use super::{EngineServices, Handler};
use crate::context::{Context, WorkflowContext, keys};
@ -44,6 +45,7 @@ pub trait CodergenBackend: Send + Sync {
emitter: &Arc<Emitter>,
sandbox: &Arc<dyn Sandbox>,
tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
cancel_token: CancellationToken,
) -> Result<CodergenResult, Error>;
/// Run a single LLM call with no tools (one_shot mode).
@ -52,6 +54,8 @@ pub trait CodergenBackend: Send + Sync {
_node: &Node,
_prompt: &str,
_system_prompt: Option<&str>,
_emitter: &Arc<Emitter>,
_stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
Err(Error::Validation(
"one_shot mode not supported by this backend".into(),
@ -297,6 +301,7 @@ impl Handler for AgentHandler {
&services.run.emitter,
&services.run.sandbox,
tool_hooks,
services.run.cancel_token(),
)
.await;
match result {
@ -307,6 +312,7 @@ impl Handler for AgentHandler {
files_touched,
last_file_touched,
}) => (text, usage, files_touched, last_file_touched),
Err(Error::Cancelled) => return Err(Error::Cancelled),
Err(e) if e.is_retryable() => {
return Err(e);
}
@ -615,6 +621,7 @@ mod tests {
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn fabro_agent::Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text:
@ -675,6 +682,7 @@ mod tests {
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn fabro_agent::Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "Done writing results.".to_string(),
@ -736,6 +744,7 @@ mod tests {
emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn fabro_agent::Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
let scope = StageScope::for_handler(context, &node.id);
emitter.emit_scoped(
@ -847,6 +856,7 @@ mod tests {
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
*self.captured_thread_id.lock().unwrap() = Some(thread_id.map(String::from));
Ok(CodergenResult::Text {
@ -899,6 +909,7 @@ mod tests {
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
*self.captured_thread_id.lock().unwrap() = Some(thread_id.map(String::from));
Ok(CodergenResult::Text {
@ -946,6 +957,7 @@ mod tests {
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
Err(Error::handler("Request timed out".to_string()))
}
@ -1093,6 +1105,7 @@ Some text in between.
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
Err(Error::Validation("bad config".to_string()))
}
@ -1133,6 +1146,7 @@ Some text in between.
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(prompt.to_string());
Ok(CodergenResult::Text {
@ -1202,6 +1216,7 @@ Some text in between.
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(prompt.to_string());
Ok(CodergenResult::Text {

13
lib/crates/fabro-workflow/src/handler/command.rs
View file

@ -109,7 +109,7 @@ impl Handler for CommandHandler {
} else {
Some(&services.env)
};
let cancel_token = services.run.sandbox_cancel_token();
let cancel_token = services.run.cancel_token().child_token();
let stage_id = stage_scope.stage_id();
let recorder = CommandLogRecorder::create(run_dir, &stage_id).await?;
let output_callback: CommandOutputCallback = {
@ -130,16 +130,14 @@ impl Handler for CommandHandler {
.sandbox
.exec_command_streaming(
&command,
timeout_ms,
Some(timeout_ms),
None,
env_vars,
cancel_token.clone(),
Some(cancel_token.clone()),
output_callback,
)
.await;
if let Some(token) = cancel_token {
token.cancel();
}
cancel_token.cancel();
let streaming = match result {
Ok(streaming) => streaming,
Err(err) => {
@ -237,7 +235,6 @@ fn tail_bytes(text: &str, max_bytes: usize) -> String {
#[cfg(test)]
mod tests {
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
use bytes::Bytes;
@ -1126,7 +1123,7 @@ mod tests {
let mut services = make_spy_services(spy.clone());
services.run = services
.run
.with_cancel_requested(Some(Arc::new(AtomicBool::new(false))));
.with_cancel_token(tokio_util::sync::CancellationToken::new());
handler
.execute(&node, &context, &graph, run_dir.path(), &services)

5
lib/crates/fabro-workflow/src/handler/fan_in.rs
View file

@ -4,6 +4,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use fabro_agent::Sandbox;
use fabro_graphviz::graph::{Graph, Node};
use tokio_util::sync::CancellationToken;
use super::agent::{CodergenBackend, CodergenResult};
use super::{EngineServices, Handler};
@ -86,6 +87,7 @@ impl Handler for FanInHandler {
&node.id,
&services.run.emitter,
&services.run.sandbox,
services.run.cancel_token(),
)
.await?
} else {
@ -223,6 +225,7 @@ async fn llm_evaluate(
node_id: &str,
emitter: &Arc<Emitter>,
sandbox: &Arc<dyn Sandbox>,
cancel_token: CancellationToken,
) -> Result<Candidate, Error> {
let results_text =
serde_json::to_string_pretty(results).unwrap_or_else(|_| results.to_string());
@ -259,6 +262,7 @@ async fn llm_evaluate(
emitter,
sandbox,
None,
cancel_token,
)
.await
{
@ -474,6 +478,7 @@ mod tests {
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
// Return text that contains the ID "branch_b"
Ok(CodergenResult::Text {

7
lib/crates/fabro-workflow/src/handler/human.rs
View file

@ -325,12 +325,7 @@ impl Handler for HumanHandler {
// 5. Handle unanswered / interrupted interview sessions.
if answer.value == AnswerValue::Interrupted {
if services
.run
.cancel_requested
.as_ref()
.is_some_and(|flag| flag.load(Ordering::SeqCst))
{
if services.run.cancel_token().is_cancelled() {
return Err(Error::Cancelled);
}
self.emit(

635
lib/crates/fabro-workflow/src/handler/llm/api.rs
View file

@ -15,6 +15,8 @@ use fabro_mcp::config::McpServerSettings;
use fabro_model::{FallbackTarget, Provider};
use fabro_types::StageId;
use tokio::sync::Mutex as TokioMutex;
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use super::super::agent::{CodergenBackend, CodergenResult};
use crate::context::keys::Fidelity;
@ -24,6 +26,98 @@ use crate::event::{Emitter, Event, StageScope};
use crate::outcome::billed_model_usage_from_llm;
use crate::steering_hub::SteeringHub;
/// Spawn a task that, when the run-level token cancels, sets the agent
/// `Session`'s interrupt reason to `Cancelled` and cancels the session token.
///
/// Factored out of `SessionCancelBridgeGuard::replace` so it can be unit-tested
/// without constructing a real `Session`.
fn spawn_bridge_task(
run_token: CancellationToken,
interrupt_reason: Arc<Mutex<Option<fabro_agent::InterruptReason>>>,
session_token: CancellationToken,
) -> JoinHandle<()> {
tokio::spawn(async move {
run_token.cancelled().await;
{
let mut guard = interrupt_reason
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if guard.is_none() {
*guard = Some(fabro_agent::InterruptReason::Cancelled);
}
}
session_token.cancel();
})
}
/// Per-invocation guard that maps a run-level `CancellationToken` to an agent
/// `Session`'s interrupt reason and cancel token.
///
/// Dropping the guard aborts the spawned bridge task so a still-cached session
/// (after success) is not left wired to a stale run token.
struct SessionCancelBridgeGuard {
handle: Option<JoinHandle<()>>,
}
impl SessionCancelBridgeGuard {
fn new() -> Self {
Self { handle: None }
}
fn replace(&mut self, run_token: CancellationToken, session: &Session) {
self.abort();
self.handle = Some(spawn_bridge_task(
run_token,
session.interrupt_reason_handle(),
session.cancel_token(),
));
}
fn abort(&mut self) {
if let Some(handle) = self.handle.take() {
handle.abort();
}
}
}
impl Drop for SessionCancelBridgeGuard {
fn drop(&mut self) {
self.abort();
}
}
/// Classification of an `fabro_agent::Error` for the API backend's `run` path.
enum AgentApiErrorDisposition {
/// Session was interrupted via cancellation; surface as `Error::Cancelled`.
Cancelled,
/// Underlying LLM error eligible for provider failover.
FailoverEligible(fabro_llm::Error),
/// Terminal error; abort the invocation with this workflow `Error`.
Terminal(Error),
}
fn classify_agent_error(err: fabro_agent::Error, allow_failover: bool) -> AgentApiErrorDisposition {
match err {
fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::Cancelled) => {
AgentApiErrorDisposition::Cancelled
}
fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::WallClockTimeout) => {
AgentApiErrorDisposition::Terminal(Error::Precondition(
"Agent session hit its wall-clock timeout".to_string(),
))
}
fabro_agent::Error::Llm(err) if allow_failover && err.failover_eligible() => {
AgentApiErrorDisposition::FailoverEligible(err)
}
fabro_agent::Error::Llm(err) => AgentApiErrorDisposition::Terminal(Error::Llm(err)),
other @ (fabro_agent::Error::SessionClosed
| fabro_agent::Error::InvalidState(_)
| fabro_agent::Error::ToolExecution(_)) => AgentApiErrorDisposition::Terminal(
Error::Precondition(format!("Agent session failed: {other}")),
),
}
}
fn build_profile(model: &str, provider: Provider) -> Box<dyn AgentProfile> {
match provider {
Provider::OpenAi => Box::new(OpenAiProfile::new(model)),
@ -305,6 +399,8 @@ impl CodergenBackend for AgentApiBackend {
node: &Node,
prompt: &str,
system_prompt: Option<&str>,
emitter: &Arc<Emitter>,
stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
let client = Client::from_source(self.source.as_ref())
.await
@ -378,14 +474,16 @@ impl CodergenBackend for AgentApiBackend {
let mut found = None;
for target in fallback_chain {
tracing::warn!(
stage = node.id.as_str(),
from_provider = from_provider.as_str(),
from_model = from_model.as_str(),
to_provider = target.provider.as_str(),
to_model = target.model.as_str(),
error = error_msg.as_str(),
"LLM provider failover (prompt)"
emitter.emit_scoped(
&Event::Failover {
stage: node.id.clone(),
from_provider: from_provider.clone(),
from_model: from_model.clone(),
to_provider: target.provider.clone(),
to_model: target.model.clone(),
error: error_msg.clone(),
},
stage_scope,
);
let max_tokens = node.max_tokens().or_else(|| {
@ -446,6 +544,7 @@ impl CodergenBackend for AgentApiBackend {
emitter: &Arc<Emitter>,
sandbox: &Arc<dyn Sandbox>,
tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
let actual_model = node.model().unwrap_or(&self.model).to_string();
let _actual_provider = node
@ -460,25 +559,36 @@ impl CodergenBackend for AgentApiBackend {
None
};
// Take a cached session if reusing, otherwise create a new one.
let mut bridge = SessionCancelBridgeGuard::new();
// Take a cached session if reusing, otherwise create a new one. Cancel
// checks bracket `Client::from_source(...)` so cancellation arriving
// during credential refresh is not lost.
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
let (mut session, is_reused) = if let Some(ref key) = reuse_key {
let existing = self.sessions.lock().unwrap().remove(key);
if let Some(s) = existing {
(s, true)
} else {
(
self.create_session(node, sandbox, tool_hooks.clone())
.await?,
false,
)
let created = self.create_session(node, sandbox, tool_hooks.clone()).await;
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
(created?, false)
}
} else {
(
self.create_session(node, sandbox, tool_hooks.clone())
.await?,
false,
)
let created = self.create_session(node, sandbox, tool_hooks.clone()).await;
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
(created?, false)
};
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
bridge.replace(cancel_token.clone(), &session);
tracing::info!(
node = %node.id,
@ -505,116 +615,174 @@ impl CodergenBackend for AgentApiBackend {
);
// Record turn count before processing so we only aggregate new usage.
let turns_before = session.history().turns().len();
if !is_reused {
session.initialize().await;
}
let mut turns_before = session.history().turns().len();
// Register with the steering hub so HTTP `POST /runs/{id}/steer`
// calls reach this session. The RAII guard below unregisters on
// every exit path (success, error, failover replace).
let stage_id = stage_scope.stage_id();
self.attach_session_to_hub(&mut session, &stage_id);
let _hub_guard = {
let hub = Arc::clone(&self.steering_hub);
let sid = stage_id.clone();
scopeguard::guard((), move |()| hub.unregister(&sid))
};
let result = session.process_input(prompt).await;
// On failover-eligible error, try fallback providers.
let result = match result {
Ok(()) => Ok(()),
Err(fabro_agent::Error::Llm(ref sdk_err))
if sdk_err.failover_eligible() && !self.fallback_chain.is_empty() =>
{
let error_msg = sdk_err.to_string();
let from_provider = self.provider.to_string();
let from_model = self.model.clone();
let mut last_err = Error::Llm(sdk_err.clone());
let mut succeeded = false;
for target in &self.fallback_chain {
emitter.emit_scoped(
&Event::Failover {
stage: node.id.clone(),
from_provider: from_provider.clone(),
from_model: from_model.clone(),
to_provider: target.provider.clone(),
to_model: target.model.clone(),
error: error_msg.clone(),
},
&stage_scope,
);
let target_provider: Provider = match target.provider.parse() {
Ok(p) => p,
Err(_) => continue,
};
let new_session = match Self::create_session_for(
&target.model,
target_provider,
node,
sandbox,
self.source.as_ref(),
&self.env,
tool_hooks.clone(),
self.mcp_servers.clone(),
)
.await
{
Ok(s) => s,
Err(e) => {
last_err = e;
continue;
}
};
session = new_session;
// Re-subscribe to forward events + track files from the new session
spawn_event_forwarder(
&session,
node.id.clone(),
stage_scope.clone(),
Arc::clone(emitter),
Arc::clone(&file_tracking),
);
// Re-register the new session's handle under the same
// stage_id (replace, no re-drain, no attached event).
self.attach_session_to_hub(&mut session, &stage_id);
session.initialize().await;
match session.process_input(prompt).await {
Ok(()) => {
succeeded = true;
break;
}
Err(fabro_agent::Error::Llm(err)) if err.failover_eligible() => {
last_err = Error::Llm(err);
}
Err(fabro_agent::Error::Llm(err)) => return Err(Error::Llm(err)),
Err(fabro_agent::Error::Interrupted(_)) => {
return Err(Error::Cancelled);
}
Err(other) => {
return Err(Error::handler(format!("Agent session failed: {other}")));
}
let allow_failover_primary = !self.fallback_chain.is_empty();
let init_result = if is_reused {
Ok(())
} else {
match session.initialize().await {
Ok(()) => Ok(()),
Err(err) => match classify_agent_error(err, allow_failover_primary) {
AgentApiErrorDisposition::Cancelled => {
bridge.abort();
return Err(Error::Cancelled);
}
}
if succeeded { Ok(()) } else { Err(last_err) }
AgentApiErrorDisposition::Terminal(err) => {
bridge.abort();
return Err(err);
}
AgentApiErrorDisposition::FailoverEligible(sdk_err) => {
Err(fabro_agent::Error::Llm(sdk_err))
}
},
}
Err(fabro_agent::Error::Llm(sdk_err)) => Err(Error::Llm(sdk_err)),
Err(fabro_agent::Error::Interrupted(_)) => Err(Error::Cancelled),
Err(other) => Err(Error::handler(format!("Agent session failed: {other}"))),
};
// On error, drop the session (don't cache failed state).
// If initialize failed with a failover-eligible error, treat as a
// process_input failover trigger; otherwise run process_input.
let result = match init_result {
Ok(()) => {
self.attach_session_to_hub(&mut session, &stage_id);
session.process_input(prompt).await
}
Err(err) => Err(err),
};
// On failover-eligible error, try fallback providers.
let result: Result<(), Error> = match result {
Ok(()) => Ok(()),
Err(err) => match classify_agent_error(err, allow_failover_primary) {
AgentApiErrorDisposition::Cancelled => {
bridge.abort();
return Err(Error::Cancelled);
}
AgentApiErrorDisposition::Terminal(err) => {
bridge.abort();
return Err(err);
}
AgentApiErrorDisposition::FailoverEligible(sdk_err) => {
let error_msg = sdk_err.to_string();
let from_provider = self.provider.to_string();
let from_model = self.model.clone();
let mut last_err = Error::Llm(sdk_err);
let mut succeeded = false;
for (index, target) in self.fallback_chain.iter().enumerate() {
emitter.emit_scoped(
&Event::Failover {
stage: node.id.clone(),
from_provider: from_provider.clone(),
from_model: from_model.clone(),
to_provider: target.provider.clone(),
to_model: target.model.clone(),
error: error_msg.clone(),
},
&stage_scope,
);
let target_provider: Provider = match target.provider.parse() {
Ok(p) => p,
Err(_) => continue,
};
// Detach the bridge from the failing session before
// refreshing credentials and building a new one.
bridge.abort();
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
let new_session_result = Self::create_session_for(
&target.model,
target_provider,
node,
sandbox,
self.source.as_ref(),
&self.env,
tool_hooks.clone(),
self.mcp_servers.clone(),
)
.await;
if cancel_token.is_cancelled() {
return Err(Error::Cancelled);
}
let new_session = match new_session_result {
Ok(s) => s,
Err(e) => {
last_err = e;
continue;
}
};
session = new_session;
bridge.replace(cancel_token.clone(), &session);
turns_before = session.history().turns().len();
// Re-subscribe to forward events + track files from the new session
spawn_event_forwarder(
&session,
node.id.clone(),
stage_scope.clone(),
Arc::clone(emitter),
Arc::clone(&file_tracking),
);
let allow_failover_next = index + 1 < self.fallback_chain.len();
if let Err(err) = session.initialize().await {
match classify_agent_error(err, allow_failover_next) {
AgentApiErrorDisposition::Cancelled => {
bridge.abort();
return Err(Error::Cancelled);
}
AgentApiErrorDisposition::Terminal(err) => {
bridge.abort();
return Err(err);
}
AgentApiErrorDisposition::FailoverEligible(sdk_err) => {
last_err = Error::Llm(sdk_err);
continue;
}
}
}
self.attach_session_to_hub(&mut session, &stage_id);
match session.process_input(prompt).await {
Ok(()) => {
succeeded = true;
break;
}
Err(err) => match classify_agent_error(err, allow_failover_next) {
AgentApiErrorDisposition::Cancelled => {
bridge.abort();
return Err(Error::Cancelled);
}
AgentApiErrorDisposition::Terminal(err) => {
bridge.abort();
return Err(err);
}
AgentApiErrorDisposition::FailoverEligible(sdk_err) => {
last_err = Error::Llm(sdk_err);
}
},
}
}
if succeeded { Ok(()) } else { Err(last_err) }
}
},
};
// On error, drop the session (don't cache failed state). The bridge's
// `Drop` will abort the spawned task on early return.
result?;
// Aggregate token usage only from new turns (prevents double-counting on
@ -657,8 +825,10 @@ impl CodergenBackend for AgentApiBackend {
(v, s.last.clone())
};
// Cache session back for reuse on success.
// Cache session back for reuse on success. Detach the bridge first so
// the cached session is not left wired to this run's cancel token.
if let Some(key) = reuse_key {
bridge.abort();
self.sessions.lock().unwrap().insert(key, session);
}
@ -698,6 +868,7 @@ impl CompletionCoordinator for SteeringCompletionCoordinator {
mod tests {
use fabro_agent::subagent::SessionFactory;
use fabro_auth::{AuthCredential, AuthDetails, VaultCredentialSource};
use fabro_llm::{Error as LlmError, ProviderErrorDetail, ProviderErrorKind};
use fabro_vault::{SecretType, Vault};
use tokio::sync::RwLock as AsyncRwLock;
@ -881,4 +1052,238 @@ mod tests {
assert_eq!(client.provider_names(), vec!["anthropic"]);
}
// --- Bridge guard tests ---
fn failover_eligible_llm_error() -> LlmError {
LlmError::Network {
message: "boom".into(),
source: None,
}
}
fn non_failover_llm_error() -> LlmError {
LlmError::Provider {
kind: ProviderErrorKind::Authentication,
detail: Box::new(ProviderErrorDetail {
message: "bad key".into(),
provider: "openai".into(),
status_code: Some(401),
error_code: None,
retry_after: None,
raw: None,
}),
}
}
#[tokio::test]
async fn spawn_bridge_task_sets_cancelled_and_cancels_session_token() {
let run_token = CancellationToken::new();
let interrupt_reason = Arc::new(Mutex::new(None));
let session_token = CancellationToken::new();
let handle = spawn_bridge_task(
run_token.clone(),
Arc::clone(&interrupt_reason),
session_token.clone(),
);
assert!(!session_token.is_cancelled());
assert!(interrupt_reason.lock().unwrap().is_none());
run_token.cancel();
handle.await.unwrap();
assert!(session_token.is_cancelled());
assert_eq!(
*interrupt_reason.lock().unwrap(),
Some(fabro_agent::InterruptReason::Cancelled)
);
}
#[tokio::test]
async fn spawn_bridge_task_preserves_existing_interrupt_reason() {
let run_token = CancellationToken::new();
let interrupt_reason = Arc::new(Mutex::new(Some(
fabro_agent::InterruptReason::WallClockTimeout,
)));
let session_token = CancellationToken::new();
let handle = spawn_bridge_task(
run_token.clone(),
Arc::clone(&interrupt_reason),
session_token.clone(),
);
run_token.cancel();
handle.await.unwrap();
// Existing reason wins; the bridge does not overwrite a wall-clock
// timeout already recorded by the session.
assert_eq!(
*interrupt_reason.lock().unwrap(),
Some(fabro_agent::InterruptReason::WallClockTimeout)
);
assert!(session_token.is_cancelled());
}
#[tokio::test]
async fn bridge_guard_drop_aborts_pending_task() {
let run_token = CancellationToken::new();
let interrupt_reason = Arc::new(Mutex::new(None));
let session_token = CancellationToken::new();
{
let mut guard = SessionCancelBridgeGuard::new();
guard.handle = Some(spawn_bridge_task(
run_token.clone(),
Arc::clone(&interrupt_reason),
session_token.clone(),
));
// guard dropped here
}
// Trigger the run token after the guard has been dropped. The aborted
// task must not write to interrupt_reason or cancel session_token.
run_token.cancel();
// Yield enough times for any errant task to run.
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert!(interrupt_reason.lock().unwrap().is_none());
assert!(!session_token.is_cancelled());
}
#[tokio::test]
async fn bridge_guard_replace_aborts_prior_task() {
// First (prior) bridge wiring.
let prior_run_token = CancellationToken::new();
let prior_interrupt_reason = Arc::new(Mutex::new(None));
let prior_session_token = CancellationToken::new();
// Second (replacement) bridge wiring.
let new_run_token = CancellationToken::new();
let new_interrupt_reason = Arc::new(Mutex::new(None));
let new_session_token = CancellationToken::new();
let mut guard = SessionCancelBridgeGuard::new();
guard.handle = Some(spawn_bridge_task(
prior_run_token.clone(),
Arc::clone(&prior_interrupt_reason),
prior_session_token.clone(),
));
// Replace with a new task pointing at different handles.
guard.handle = {
// Manually mirror `replace` semantics: abort then install.
if let Some(h) = guard.handle.take() {
h.abort();
}
Some(spawn_bridge_task(
new_run_token.clone(),
Arc::clone(&new_interrupt_reason),
new_session_token.clone(),
))
};
// Cancelling the prior run token must not affect anything because the
// prior task was aborted by `replace`.
prior_run_token.cancel();
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert!(prior_interrupt_reason.lock().unwrap().is_none());
assert!(!prior_session_token.is_cancelled());
// The replacement task must still be alive and react to its own token.
new_run_token.cancel();
guard.handle.take().unwrap().await.unwrap();
assert_eq!(
*new_interrupt_reason.lock().unwrap(),
Some(fabro_agent::InterruptReason::Cancelled)
);
assert!(new_session_token.is_cancelled());
}
// --- classify_agent_error tests ---
#[test]
fn classify_interrupted_cancelled_is_cancelled() {
let err = fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::Cancelled);
assert!(matches!(
classify_agent_error(err, true),
AgentApiErrorDisposition::Cancelled
));
}
#[test]
fn classify_interrupted_wall_clock_is_terminal_precondition() {
let err = fabro_agent::Error::Interrupted(fabro_agent::InterruptReason::WallClockTimeout);
match classify_agent_error(err, true) {
AgentApiErrorDisposition::Terminal(Error::Precondition(msg)) => {
assert!(msg.contains("wall-clock"));
}
_ => panic!("expected Terminal(Error::Precondition) for WallClockTimeout"),
}
}
#[test]
fn classify_failover_eligible_llm_returns_failover_when_allowed() {
let err = fabro_agent::Error::Llm(failover_eligible_llm_error());
assert!(matches!(
classify_agent_error(err, true),
AgentApiErrorDisposition::FailoverEligible(_)
));
}
#[test]
fn classify_failover_eligible_llm_returns_terminal_when_not_allowed() {
let err = fabro_agent::Error::Llm(failover_eligible_llm_error());
match classify_agent_error(err, false) {
AgentApiErrorDisposition::Terminal(Error::Llm(_)) => {}
_ => panic!("expected Terminal(Error::Llm) when failover disallowed"),
}
}
#[test]
fn classify_non_failover_eligible_llm_is_terminal_llm() {
let err = fabro_agent::Error::Llm(non_failover_llm_error());
match classify_agent_error(err, true) {
AgentApiErrorDisposition::Terminal(Error::Llm(_)) => {}
_ => panic!("expected Terminal(Error::Llm) for non-failover-eligible LLM error"),
}
}
#[test]
fn classify_session_closed_is_terminal_precondition() {
let err = fabro_agent::Error::SessionClosed;
match classify_agent_error(err, true) {
AgentApiErrorDisposition::Terminal(Error::Precondition(message)) => {
assert!(message.contains("Agent session failed"));
}
_ => panic!("expected Terminal(Error::Precondition) for SessionClosed"),
}
}
#[test]
fn classify_invalid_state_is_terminal_precondition() {
let err = fabro_agent::Error::InvalidState("oops".into());
match classify_agent_error(err, true) {
AgentApiErrorDisposition::Terminal(Error::Precondition(message)) => {
assert!(message.contains("Agent session failed"));
}
_ => panic!("expected Terminal(Error::Precondition) for InvalidState"),
}
}
#[test]
fn classify_tool_execution_is_terminal_precondition() {
let err = fabro_agent::Error::ToolExecution("tool blew up".into());
match classify_agent_error(err, true) {
AgentApiErrorDisposition::Terminal(Error::Precondition(message)) => {
assert!(message.contains("Agent session failed"));
}
_ => panic!("expected Terminal(Error::Precondition) for ToolExecution"),
}
}
}

601
lib/crates/fabro-workflow/src/handler/llm/cli.rs
View file

@ -1,16 +1,37 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use fabro_agent::Sandbox;
use fabro_agent::sandbox::ExecResult;
use fabro_agent::{Sandbox, shell_quote};
use fabro_auth::{CliAgentKind, CredentialResolver, CredentialUsage, ResolvedCredential};
use fabro_graphviz::graph::Node;
use fabro_llm::types::TokenCounts;
use fabro_model::Provider;
use fabro_types::CommandTermination;
use fabro_types::{CommandOutputStream, CommandTermination};
use fabro_util::time::elapsed_ms;
use tokio::time::sleep;
use tokio_util::sync::CancellationToken;
/// Returns up to the last `n` characters of `s`, preserving char boundaries.
fn tail_chars(s: &str, n: usize) -> String {
let total = s.chars().count();
if total <= n {
return s.to_string();
}
s.chars().skip(total - n).collect()
}
/// Build a "<stderr-tail>\nstdout: <stdout-tail>" detail string for CLI failure
/// messages, falling back to the original command when both streams are empty.
fn cli_failure_detail(stdout: &str, stderr: &str, command: &str) -> String {
let stderr_tail = tail_chars(stderr, 500);
let stdout_tail = tail_chars(stdout, 500);
match (stderr_tail.is_empty(), stdout_tail.is_empty()) {
(false, false) => format!("{stderr_tail}\nstdout: {stdout_tail}"),
(false, true) => stderr_tail,
(true, false) => format!("stdout: {stdout_tail}"),
(true, true) => format!("command: {command}"),
}
}
use super::super::agent::{CodergenBackend, CodergenResult};
use crate::context::Context;
@ -66,6 +87,7 @@ async fn ensure_cli(
provider: Provider,
sandbox: &Arc<dyn Sandbox>,
emitter: &Arc<Emitter>,
cancel_token: &CancellationToken,
) -> Result<(), Error> {
let start = std::time::Instant::now();
let cli_name = cli.name();
@ -84,7 +106,7 @@ async fn ensure_cli(
30_000,
None,
None,
None,
Some(cancel_token.child_token()),
)
.await
.map_err(|e| {
@ -112,7 +134,13 @@ async fn ensure_cli(
cli.npm_package()
);
let install_result = sandbox
.exec_command(&install_cmd, 180_000, None, None, None)
.exec_command(
&install_cmd,
180_000,
None,
None,
Some(cancel_token.child_token()),
)
.await
.map_err(|e| Error::handler_with_source(format!("Failed to install {cli_name}"), &e))?;
@ -161,9 +189,11 @@ pub fn is_cli_only_model(model: &str) -> bool {
/// is piped into the command's stdin via `cat`.
#[must_use]
pub fn cli_command_for_provider(provider: Provider, model: &str, prompt_file: &str) -> String {
let prompt_file = shell_quote(prompt_file);
let model_flag = if model.is_empty() {
String::new()
} else {
let model = shell_quote(model);
match provider {
Provider::OpenAi
| Provider::Gemini
@ -362,14 +392,6 @@ pub fn parse_cli_response(provider: Provider, output: &str) -> Option<CliRespons
}
}
/// Escape a value for safe embedding inside single quotes in a shell command.
fn shell_quote(val: &str) -> String {
shlex::try_quote(val).map_or_else(
|_| format!("'{}'", val.replace('\'', "'\\''")),
std::borrow::Cow::into_owned,
)
}
/// CLI backend that invokes external CLI tools (claude, codex, gemini) via
/// `exec_command()`.
pub struct AgentCliBackend {
@ -477,6 +499,7 @@ impl CodergenBackend for AgentCliBackend {
emitter: &Arc<Emitter>,
sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
// 1. Snapshot git state before the CLI run
let files_before = self.detect_changed_files(sandbox).await;
@ -485,9 +508,6 @@ impl CodergenBackend for AgentCliBackend {
let run_id = uuid::Uuid::new_v4().to_string();
let tmp_prefix = format!("/tmp/fabro_cli_{run_id}");
let prompt_path = format!("{tmp_prefix}_prompt.txt");
let stdout_path = format!("{tmp_prefix}_stdout.log");
let stderr_path = format!("{tmp_prefix}_stderr.log");
let exit_code_path = format!("{tmp_prefix}_exit_code");
let env_path = format!("{tmp_prefix}_env.sh");
sandbox
@ -504,7 +524,7 @@ impl CodergenBackend for AgentCliBackend {
// Ensure the CLI tool is installed in the sandbox
let cli = AgentCli::for_provider(provider);
ensure_cli(cli, provider, sandbox, emitter).await?;
ensure_cli(cli, provider, sandbox, emitter, &cancel_token).await?;
let command = cli_command_for_provider(provider, model, &prompt_path);
let stage_scope = StageScope::for_handler(context, &node.id);
@ -521,11 +541,8 @@ impl CodergenBackend for AgentCliBackend {
);
// Forward provider API key and custom env vars so the CLI tool can
// authenticate. Build a HashMap to pass via exec_command's env_vars
// parameter — this prepends `export` statements directly into the
// base64-encoded command, avoiding filesystem-to-process race
// conditions that can occur when writing an env file via the fs API and
// sourcing it via the process API.
// authenticate. Resolve credentials and run any pre-login command
// before the main CLI invocation.
let cli_agent = match cli {
AgentCli::Claude => CliAgentKind::Claude,
AgentCli::Codex => CliAgentKind::Codex,
@ -541,7 +558,13 @@ impl CodergenBackend for AgentCliBackend {
};
if let Some(login_cmd) = &cli_credential.login_command {
let login_result = sandbox
.exec_command(login_cmd, 30_000, None, None, None)
.exec_command(
login_cmd,
30_000,
None,
None,
Some(cancel_token.child_token()),
)
.await
.map_err(|e| Error::handler_with_source("codex login failed", &e))?;
if !login_result.is_success() {
@ -566,127 +589,183 @@ impl CodergenBackend for AgentCliBackend {
launch_env.insert(name.clone(), val.clone());
}
// Also write env file as fallback for commands that source it (e.g. ensure_cli
// PATH)
// Write env file so the inner shell that runs the CLI command picks up
// PATH and provider env vars; we still pass `launch_env` to
// `exec_command_streaming` for parity.
let mut env_lines: Vec<String> = vec!["export PATH=\"$HOME/.local/bin:$PATH\"".to_string()];
env_lines.extend(
launch_env
.iter()
.map(|(k, v)| format!("export {k}={}", shell_quote(v))),
);
{
sandbox
.write_file(&env_path, &env_lines.join("\n"))
.await
.map_err(|e| Error::handler_with_source("Failed to write env file", &e))?;
}
sandbox
.write_file(&env_path, &env_lines.join("\n"))
.await
.map_err(|e| Error::handler_with_source("Failed to write env file", &e))?;
// 3a. Disable auto-stop so the sandbox stays alive during long CLI runs
// Disable auto-stop so the sandbox stays alive during long CLI runs.
if let Err(e) = sandbox.set_autostop_interval(0).await {
tracing::warn!("Failed to disable sandbox auto-stop: {e}");
}
// 3b. Launch CLI command in background (env file is always written)
let inner_command = format!(". {env_path} && {command}");
// Use setsid (if available) to create a new session so the child process is
// fully detached from the shell. Without this, Daytona's POST /process/execute
// blocks until ALL descendant processes exit, causing a 60s HTTP timeout.
// $SID is empty on macOS (where setsid doesn't exist but isn't needed since
// the local exec implementation doesn't wait for grandchildren).
let bg_command = format!(
"SID=$(command -v setsid || true)\n$SID sh -c '{inner_command} > {stdout_path} 2>{stderr_path}; echo $? > {exit_code_path}' </dev/null >/dev/null 2>&1 &\necho $!"
);
let launch_start = std::time::Instant::now();
// Stream the CLI command directly: the previous detached `setsid &`
// launcher could not be cancelled mid-flight. By running through
// `exec_command_streaming` the run-level cancel token (and node
// timeout, when set) terminate the CLI and its descendants.
let outer_command = format!(". {} && {command}", shell_quote(&env_path));
// Use a synchronous Mutex: each callback invocation only does a short
// `extend_from_slice` with no awaits while the lock is held, so an
// async Mutex would just add per-chunk scheduling overhead.
let stdout_buffer: Arc<Mutex<Vec<u8>>> = Arc::new(Mutex::new(Vec::new()));
let stderr_buffer: Arc<Mutex<Vec<u8>>> = Arc::new(Mutex::new(Vec::new()));
let stdout_buf_cb = Arc::clone(&stdout_buffer);
let stderr_buf_cb = Arc::clone(&stderr_buffer);
let emitter_for_callback = Arc::clone(emitter);
let output_callback: fabro_agent::CommandOutputCallback = Arc::new(move |stream, bytes| {
let stdout_buf = Arc::clone(&stdout_buf_cb);
let stderr_buf = Arc::clone(&stderr_buf_cb);
let emitter = Arc::clone(&emitter_for_callback);
Box::pin(async move {
// Touch the stall watchdog whenever the CLI emits output
// so long-running invocations don't trip stall timeout.
emitter.touch();
let buf = match stream {
CommandOutputStream::Stdout => stdout_buf,
CommandOutputStream::Stderr => stderr_buf,
};
buf.lock()
.expect("CLI output buffer mutex poisoned")
.extend_from_slice(&bytes);
Ok(())
})
});
let launch_env_ref = if launch_env.is_empty() {
None
} else {
Some(&launch_env)
};
let launch_result = sandbox
.exec_command(&bg_command, 30_000, None, launch_env_ref, None)
.await
.map_err(|e| Error::handler_with_source("Failed to launch CLI command", &e))?;
let pid = launch_result.stdout.trim();
tracing::info!(pid, "CLI process launched in background");
let timeout_ms = node.timeout().map(crate::millis_u64);
let invocation_token = cancel_token.child_token();
let launch_start = std::time::Instant::now();
let streaming_result = sandbox
.exec_command_streaming(
&outer_command,
timeout_ms,
None,
launch_env_ref,
Some(invocation_token.clone()),
output_callback,
)
.await;
// 3c. Poll for completion
let poll_command =
format!("[ -f {exit_code_path} ] && cat {exit_code_path} || echo running");
let poll_interval = self.poll_interval;
let exit_code: i32 = loop {
sleep(poll_interval).await;
emitter.touch(); // keep the stall watchdog alive while polling
let poll_result = sandbox
.exec_command(&poll_command, 30_000, None, None, None)
.await
.map_err(|e| Error::handler_with_source("Failed to poll CLI command", &e))?;
let status = poll_result.stdout.trim();
if status != "running" {
break status.parse::<i32>().unwrap_or(-1);
let cleanup_temp_files = || {
let sandbox = Arc::clone(sandbox);
let cleanup_cmd = format!("rm -f {}_*", shell_quote(&tmp_prefix));
async move {
let _ = sandbox
.exec_command(&cleanup_cmd, 30_000, None, None, None)
.await;
}
};
// 3d. Read results
let duration_ms = u64::try_from(launch_start.elapsed().as_millis()).unwrap_or(u64::MAX);
let stdout_result = sandbox
.exec_command(&format!("cat {stdout_path}"), 60_000, None, None, None)
.await
.map_err(|e| Error::handler_with_source("Failed to read stdout", &e))?;
let stderr_result = sandbox
.exec_command(&format!("cat {stderr_path}"), 60_000, None, None, None)
.await
.map_err(|e| Error::handler_with_source("Failed to read stderr", &e))?;
let result = ExecResult {
stdout: stdout_result.stdout,
stderr: stderr_result.stdout,
exit_code: Some(exit_code),
termination: CommandTermination::Exited,
duration_ms,
let streaming = match streaming_result {
Ok(streaming) => streaming,
Err(err) => {
cleanup_temp_files().await;
return Err(Error::handler_with_source(
"Failed to run CLI command",
&err,
));
}
};
emitter.emit_scoped(
&Event::AgentCliCompleted {
node_id: node.id.clone(),
stdout: result.stdout.clone(),
stderr: result.stderr.clone(),
exit_code: result.exit_code.unwrap_or(-1),
duration_ms: result.duration_ms,
},
&stage_scope,
);
let result = streaming.result;
// Prefer the buffered streaming output (live chunks); fall back to the
// result struct for sandboxes that bundle output at the end.
let buffered_stdout = {
let buf = stdout_buffer
.lock()
.expect("CLI stdout buffer mutex poisoned");
String::from_utf8_lossy(&buf).into_owned()
};
let buffered_stderr = {
let buf = stderr_buffer
.lock()
.expect("CLI stderr buffer mutex poisoned");
String::from_utf8_lossy(&buf).into_owned()
};
let stdout = if buffered_stdout.is_empty() {
result.stdout.clone()
} else {
buffered_stdout
};
let stderr = if buffered_stderr.is_empty() {
result.stderr.clone()
} else {
buffered_stderr
};
let duration_ms = elapsed_ms(launch_start);
// 3e. Cleanup temp files
let _ = sandbox
.exec_command(&format!("rm -f {tmp_prefix}_*"), 30_000, None, None, None)
.await;
match result.termination {
CommandTermination::Cancelled => {
emitter.emit_scoped(
&Event::AgentCliCancelled {
node_id: node.id.clone(),
stdout: stdout.clone(),
stderr: stderr.clone(),
duration_ms,
},
&stage_scope,
);
cleanup_temp_files().await;
return Err(Error::Cancelled);
}
CommandTermination::TimedOut => {
emitter.emit_scoped(
&Event::AgentCliTimedOut {
node_id: node.id.clone(),
stdout: stdout.clone(),
stderr: stderr.clone(),
duration_ms,
},
&stage_scope,
);
cleanup_temp_files().await;
let detail = cli_failure_detail(&stdout, &stderr, &command);
return Err(Error::handler(format!(
"CLI command timed out after {duration_ms} ms: {detail}"
)));
}
CommandTermination::Exited => {
emitter.emit_scoped(
&Event::AgentCliCompleted {
node_id: node.id.clone(),
stdout: stdout.clone(),
stderr: stderr.clone(),
exit_code: result.exit_code.unwrap_or(-1),
duration_ms,
},
&stage_scope,
);
}
}
if !result.is_success() {
let tail = |s: &str, n: usize| -> String {
s.chars()
.rev()
.take(n)
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect()
};
let stderr_tail = tail(&result.stderr, 500);
let stdout_tail = tail(&result.stdout, 500);
let detail = match (stderr_tail.is_empty(), stdout_tail.is_empty()) {
(false, false) => format!("{stderr_tail}\nstdout: {stdout_tail}"),
(false, true) => stderr_tail,
(true, false) => format!("stdout: {stdout_tail}"),
(true, true) => format!("command: {command}"),
};
// Cleanup temp files (Exited path).
cleanup_temp_files().await;
let exited_success =
result.termination == CommandTermination::Exited && result.exit_code == Some(0);
if !exited_success {
let detail = cli_failure_detail(&stdout, &stderr, &command);
return Err(Error::handler(format!(
"CLI command exited with code {}: {detail}",
result.display_exit_code(),
result
.exit_code
.map_or_else(|| "<unknown>".to_string(), |c| c.to_string()),
)));
}
// 4. Parse the CLI output
let parsed = parse_cli_response(provider, &result.stdout)
let parsed = parse_cli_response(provider, &stdout)
.ok_or_else(|| Error::handler("Failed to parse CLI output".to_string()))?;
// 5. Detect changed files
@ -700,10 +779,7 @@ impl CodergenBackend for AgentCliBackend {
let last_file_touched = if files_touched.is_empty() {
None
} else {
let quoted_files: Vec<String> = files_touched
.iter()
.filter_map(|f| shlex::try_quote(f).ok().map(std::borrow::Cow::into_owned))
.collect();
let quoted_files: Vec<String> = files_touched.iter().map(|f| shell_quote(f)).collect();
let cmd = format!("ls -t {} | head -1", quoted_files.join(" "));
if let Ok(result) = sandbox.exec_command(&cmd, 5_000, None, None, None).await {
let trimmed = result.stdout.trim().to_string();
@ -789,17 +865,32 @@ impl CodergenBackend for BackendRouter {
emitter: &Arc<Emitter>,
sandbox: &Arc<dyn Sandbox>,
tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
if self.should_use_cli(node) {
self.cli_backend
.run(
node, prompt, context, thread_id, emitter, sandbox, tool_hooks,
node,
prompt,
context,
thread_id,
emitter,
sandbox,
tool_hooks,
cancel_token,
)
.await
} else {
self.api_backend
.run(
node, prompt, context, thread_id, emitter, sandbox, tool_hooks,
node,
prompt,
context,
thread_id,
emitter,
sandbox,
tool_hooks,
cancel_token,
)
.await
}
@ -810,9 +901,13 @@ impl CodergenBackend for BackendRouter {
node: &Node,
prompt: &str,
system_prompt: Option<&str>,
emitter: &Arc<Emitter>,
stage_scope: &StageScope,
) -> Result<CodergenResult, Error> {
// CLI backend doesn't support one_shot, always route to API
self.api_backend.one_shot(node, prompt, system_prompt).await
self.api_backend
.one_shot(node, prompt, system_prompt, emitter, stage_scope)
.await
}
}
@ -820,6 +915,7 @@ impl CodergenBackend for BackendRouter {
mod tests {
use std::path::Path;
use fabro_agent::sandbox::ExecResult;
use fabro_graphviz::graph::AttrValue;
use super::*;
@ -999,7 +1095,14 @@ mod tests {
));
let emitter = Arc::new(Emitter::default());
let result = ensure_cli(AgentCli::Claude, Provider::Anthropic, &sandbox, &emitter).await;
let result = ensure_cli(
AgentCli::Claude,
Provider::Anthropic,
&sandbox,
&emitter,
&CancellationToken::new(),
)
.await;
assert!(result.is_ok());
let commands = commands.lock().unwrap();
@ -1020,7 +1123,14 @@ mod tests {
));
let emitter = Arc::new(Emitter::default());
let result = ensure_cli(AgentCli::Claude, Provider::Anthropic, &sandbox, &emitter).await;
let result = ensure_cli(
AgentCli::Claude,
Provider::Anthropic,
&sandbox,
&emitter,
&CancellationToken::new(),
)
.await;
assert!(result.is_ok());
let commands = commands.lock().unwrap();
@ -1045,7 +1155,14 @@ mod tests {
move |event| events.lock().unwrap().push(event.clone())
});
let result = ensure_cli(AgentCli::Claude, Provider::Anthropic, &sandbox, &emitter).await;
let result = ensure_cli(
AgentCli::Claude,
Provider::Anthropic,
&sandbox,
&emitter,
&CancellationToken::new(),
)
.await;
assert!(result.is_err());
let error = result.unwrap_err().to_string();
assert!(error.contains("install exited with code 1"));
@ -1276,6 +1393,7 @@ mod tests {
_emitter: &Arc<Emitter>,
_sandbox: &Arc<dyn Sandbox>,
_tool_hooks: Option<Arc<dyn fabro_agent::ToolHookCallback>>,
_cancel_token: CancellationToken,
) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "stub".to_string(),
@ -1285,4 +1403,233 @@ mod tests {
})
}
}
/// Sandbox stub whose `exec_command_streaming` returns a configurable
/// `CommandTermination` so we can exercise the cancel/timeout paths in
/// `AgentCliBackend::run` without spawning real processes.
struct StreamingCliMock {
commands: Arc<Mutex<Vec<String>>>,
termination: CommandTermination,
exit_code: Option<i32>,
}
#[async_trait]
impl Sandbox for StreamingCliMock {
async fn read_file(
&self,
_path: &str,
_offset: Option<usize>,
_limit: Option<usize>,
) -> fabro_sandbox::Result<String> {
Ok(String::new())
}
async fn write_file(&self, _path: &str, _content: &str) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn delete_file(&self, _path: &str) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn file_exists(&self, _path: &str) -> fabro_sandbox::Result<bool> {
Ok(false)
}
async fn list_directory(
&self,
_path: &str,
_depth: Option<usize>,
) -> fabro_sandbox::Result<Vec<fabro_agent::sandbox::DirEntry>> {
Ok(vec![])
}
async fn exec_command(
&self,
command: &str,
_timeout_ms: u64,
_working_dir: Option<&str>,
_env_vars: Option<&std::collections::HashMap<String, String>>,
_cancel_token: Option<CancellationToken>,
) -> fabro_sandbox::Result<ExecResult> {
self.commands.lock().unwrap().push(command.to_string());
// Default: success for git/version/cat/rm/ls.
if command.contains("--version") {
return Ok(ok_result());
}
Ok(ExecResult {
stdout: String::new(),
stderr: String::new(),
exit_code: Some(0),
termination: CommandTermination::Exited,
duration_ms: 1,
})
}
async fn exec_command_streaming(
&self,
command: &str,
_timeout_ms: Option<u64>,
_working_dir: Option<&str>,
_env_vars: Option<&std::collections::HashMap<String, String>>,
_cancel_token: Option<CancellationToken>,
_output_callback: fabro_agent::CommandOutputCallback,
) -> fabro_sandbox::Result<fabro_sandbox::ExecStreamingResult> {
self.commands.lock().unwrap().push(command.to_string());
Ok(fabro_sandbox::ExecStreamingResult {
result: ExecResult {
stdout: String::new(),
stderr: String::new(),
exit_code: self.exit_code,
termination: self.termination,
duration_ms: 5,
},
streams_separated: true,
live_streaming: true,
})
}
async fn grep(
&self,
_pattern: &str,
_path: &str,
_options: &fabro_agent::sandbox::GrepOptions,
) -> fabro_sandbox::Result<Vec<String>> {
Ok(vec![])
}
async fn glob(
&self,
_pattern: &str,
_path: Option<&str>,
) -> fabro_sandbox::Result<Vec<String>> {
Ok(vec![])
}
async fn download_file_to_local(&self, _: &str, _: &Path) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn upload_file_from_local(&self, _: &Path, _: &str) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn initialize(&self) -> fabro_sandbox::Result<()> {
Ok(())
}
async fn cleanup(&self) -> fabro_sandbox::Result<()> {
Ok(())
}
fn working_directory(&self) -> &str {
"/workspace"
}
fn platform(&self) -> &str {
"linux"
}
fn os_version(&self) -> String {
"Ubuntu 22.04".into()
}
async fn set_autostop_interval(&self, _minutes: i32) -> fabro_sandbox::Result<()> {
Ok(())
}
}
fn collect_events(emitter: &Arc<Emitter>) -> Arc<Mutex<Vec<fabro_types::RunEvent>>> {
let events = Arc::new(Mutex::new(Vec::new()));
let events_clone = Arc::clone(&events);
emitter.on_event(move |event| events_clone.lock().unwrap().push(event.clone()));
events
}
#[tokio::test]
async fn agent_cli_backend_run_emits_cancelled_event_and_returns_cancelled() {
let commands = Arc::new(Mutex::new(Vec::new()));
let sandbox: Arc<dyn Sandbox> = Arc::new(StreamingCliMock {
commands: Arc::clone(&commands),
termination: CommandTermination::Cancelled,
exit_code: None,
});
let backend = AgentCliBackend::new_from_env("claude-opus-4-6".into(), Provider::Anthropic);
let node = Node::new("step");
let context = Context::new();
let emitter = Arc::new(Emitter::default());
let events = collect_events(&emitter);
let result = backend
.run(
&node,
"Do something",
&context,
None,
&emitter,
&sandbox,
None,
CancellationToken::new(),
)
.await;
let Err(err) = result else {
panic!("cancelled streaming should bubble Error::Cancelled");
};
assert!(matches!(err, Error::Cancelled));
let events = events.lock().unwrap();
let names: Vec<String> = events
.iter()
.map(|e| e.body.event_name().to_string())
.collect();
assert!(
names.iter().any(|n| n == "agent.cli.cancelled"),
"expected agent.cli.cancelled, got events: {names:?}"
);
assert!(
!names.iter().any(|n| n == "agent.cli.completed"),
"should not emit agent.cli.completed on cancellation"
);
// Cleanup `rm -f` ran.
let cmds = commands.lock().unwrap();
assert!(
cmds.iter().any(|c| c.starts_with("rm -f /tmp/fabro_cli_")),
"expected temp cleanup, got commands: {cmds:?}"
);
}
#[tokio::test]
async fn agent_cli_backend_run_emits_timed_out_event_and_returns_handler_error() {
let commands = Arc::new(Mutex::new(Vec::new()));
let sandbox: Arc<dyn Sandbox> = Arc::new(StreamingCliMock {
commands: Arc::clone(&commands),
termination: CommandTermination::TimedOut,
exit_code: None,
});
let backend = AgentCliBackend::new_from_env("claude-opus-4-6".into(), Provider::Anthropic);
let node = Node::new("step");
let context = Context::new();
let emitter = Arc::new(Emitter::default());
let events = collect_events(&emitter);
let result = backend
.run(
&node,
"Do something slow",
&context,
None,
&emitter,
&sandbox,
None,
CancellationToken::new(),
)
.await;
let Err(err) = result else {
panic!("timeout streaming should produce a handler error");
};
assert!(
matches!(err, Error::Handler { .. }),
"expected handler error on timeout, got {err:?}"
);
let events = events.lock().unwrap();
let names: Vec<String> = events
.iter()
.map(|e| e.body.event_name().to_string())
.collect();
assert!(
names.iter().any(|n| n == "agent.cli.timed_out"),
"expected agent.cli.timed_out, got events: {names:?}"
);
assert!(
!names.iter().any(|n| n == "agent.cli.completed"),
"should not emit agent.cli.completed on timeout"
);
}
}

17
lib/crates/fabro-workflow/src/handler/manager_loop.rs
View file

@ -1,7 +1,6 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use async_trait::async_trait;
@ -197,13 +196,12 @@ impl Handler for SubWorkflowHandler {
let child_logs = run_dir.join(format!("stages/{}@{visit}/child", node.id));
let _ = fs::create_dir_all(&child_logs).await;
let cancel_token = Arc::new(AtomicBool::new(false));
let child_cancel = Arc::clone(&cancel_token);
let child_run_token = services.run.cancel_token().child_token();
let child_run_options = RunOptions {
settings: WorkflowSettings::default(),
run_dir: child_logs,
cancel_token: Some(cancel_token),
cancel_token: child_run_token.clone(),
// Child workflows are part of the parent run's event stream.
run_id: services.run.emitter.run_id(),
labels: HashMap::new(),
@ -246,11 +244,14 @@ impl Handler for SubWorkflowHandler {
.map_err(|err| Error::engine(err.to_string()))?;
let artifact_store = ArtifactStore::new(object_store, "artifacts");
// Spawn child engine
// Spawn child engine. Child runs receive a derived cancel token from
// the parent run; parent cancellation propagates parent-to-child via
// `child_token()`, but child cancellation does not cancel the parent.
let child_run_token_for_services = child_run_token.clone();
let mut child_handle = tokio::spawn(async move {
let child_run = parent_run
.with_run_store(run_store.into())
.with_cancel_requested(None);
.with_cancel_token(child_run_token_for_services);
let initialized = Initialized {
graph: child_graph,
source: String::new(),
@ -319,7 +320,7 @@ impl Handler for SubWorkflowHandler {
if !stop_condition.is_empty() {
let dummy_outcome = Outcome::success();
if evaluate_condition(stop_condition, &dummy_outcome, context) {
child_cancel.store(true, Ordering::Relaxed);
child_run_token.cancel();
// Give child a moment to wind down
let _ = timeout(
Duration::from_millis(100),
@ -337,7 +338,7 @@ impl Handler for SubWorkflowHandler {
}
// Max cycles exceeded — cancel child
child_cancel.store(true, Ordering::Relaxed);
child_run_token.cancel();
let _ = timeout(Duration::from_millis(100), &mut child_handle).await;
Ok(Outcome::fail_classify(format!(

1
lib/crates/fabro-workflow/src/handler/mod.rs
View file

@ -23,7 +23,6 @@ use fabro_interview::Interviewer;
use crate::context::Context;
use crate::error::Error;
use crate::outcome::{Outcome, OutcomeExt};
pub(crate) use crate::services::sandbox_cancel_token;
pub use crate::services::{EngineServices, RunServices};
/// The handler interface for node execution.

Some files were not shown because too many files have changed in this diff Show more